From noreply at buildbot.pypy.org  Wed Oct  2 17:01:30 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  2 Oct 2013 17:01:30 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: in-progress
Message-ID: <20131002150130.8DC1A1C2F76@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5064:89851c7bf6fa
Date: 2013-10-02 17:01 +0200
http://bitbucket.org/pypy/extradoc/changeset/89851c7bf6fa/

Log:	in-progress

diff --git a/talk/pyconza2013/talk.rst b/talk/pyconza2013/talk.rst
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2013/talk.rst
@@ -0,0 +1,220 @@
+
+=======================================
+Software Transactional Memory with PyPy
+=======================================
+
+
+Introduction
+------------
+
+* what is PyPy: an alternative implementation of Python
+
+* main focus is on speed
+
+
+Introduction
+------------
+
+.. image: speed.png
+
+
+SQL Databases by example
+------------------------
+
+::
+
+    BEGIN TRANSACTION;
+    SELECT * FROM ...;
+    UPDATE ...;
+    COMMIT;
+
+
+Python by example
+-----------------
+
+::
+
+    ..
+    x = obj.value
+    obj.value = x + 1
+    ..
+
+
+Python by example
+-----------------
+
+::
+
+    begin_transaction()
+    x = obj.value
+    obj.value = x + 1
+    commit_transaction()
+
+
+Python by example
+-----------------
+
+::
+
+    with atomic:
+        x = obj.value
+        obj.value = x + 1
+
+
+Python by example
+-----------------
+
+::
+
+    with the_lock:
+        x = obj.value
+        obj.value = x + 1
+
+
+Locks != Transactions
+---------------------
+
+::
+
+    BEGIN TRANSACTION;    BEGIN TRANSACTION;    BEGIN..
+    SELECT * FROM ...;    SELECT * FROM ...;    SELEC..
+    UPDATE ...;           UPDATE ...;           UPDAT..
+    COMMIT;               COMMIT;               COMMI..
+
+
+Locks != Transactions
+---------------------
+
+::
+
+    with the_lock:        with the_lock:        with ..
+      x = obj.val           x = obj.val           x =..
+      obj.val = x + 1       obj.val = x + 1       obj..
+
+
+Locks != Transactions
+---------------------
+
+::
+
+    with atomic:          with atomic:          with ..
+      x = obj.val           x = obj.val           x =..
+      obj.val = x + 1       obj.val = x + 1       obj..
+
+
+STM
+---
+
+* Transactional Memory
+
+* advanced magic (but not more so than databases)
+
+
+STM versus HTM
+--------------
+
+* Software versus Hardware
+
+* CPU hardware specially to avoid the high overhead
+
+* too limited for now
+
+
+Example 1
+---------
+
+::
+
+  def apply_interest_rate(self):
+     self.balance *= 1.05
+
+  for account in all_accounts:
+     account.apply_interest_rate()
+
+
+Example 1
+---------
+
+::
+
+  def apply_interest_rate(self):
+     self.balance *= 1.05
+
+  for account in all_accounts:
+     add_task(account.apply_interest_rate)
+  run_tasks()
+
+
+Internally
+----------
+
+* `run_all_tasks()` manages a pool of threads
+
+* each thread runs tasks in a `with atomic`
+
+
+Example 2
+---------
+
+::
+
+  def next_iteration(all_trains):
+     for train in all_trains:
+        start_time = ...
+        for othertrain in train.dependencies:
+           if ...:
+              start_time = ...
+        train.start_time = start_time
+
+
+Example 2
+---------
+
+::
+
+  def compute(train):
+     ...
+
+  def next_iteration(all_trains):
+     for train in all_trains:
+        add_task(compute, train)
+     run_all_tasks()
+
+
+By the way
+----------
+
+* STM replaces the GIL
+
+* any existing multithreaded program runs on multiple cores
+
+
+Current status
+--------------
+
+* 
+
+
+
+User feedback
+-------------
+
+::
+
+  Detected conflict:
+    File "foo.py", line 17, in walk
+      if node.left not in seen:
+  Transaction aborted, 0.000047 seconds lost
+
+
+User feedback
+-------------
+
+::
+
+  Forced inevitable:
+    File "foo.py", line 19, in walk
+      print >> log, logentry
+  Transaction blocked others for 0.xx seconds
+
+(not implemented yet)

From noreply at buildbot.pypy.org  Thu Oct  3 00:34:52 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu,  3 Oct 2013 00:34:52 +0200 (CEST)
Subject: [pypy-commit] pypy remove-intlong-smm: workaround the lack of a
 GetSetIndirectProperty for now (fix translation)
Message-ID: <20131002223452.509A61C2F76@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: remove-intlong-smm
Changeset: r67145:6c5a18b6acf1
Date: 2013-10-02 15:33 -0700
http://bitbucket.org/pypy/pypy/changeset/6c5a18b6acf1/

Log:	workaround the lack of a GetSetIndirectProperty for now (fix
	translation)

diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -360,6 +360,7 @@
             val >>= 1
         return space.wrap(bits)
 
+    """
     def descr_get_numerator(self, space):
         return space.int(self)
 
@@ -371,6 +372,20 @@
 
     def descr_get_imag(self, space):
         return space.wrap(0)
+        """
+
+# XXX:
+def descr_get_numerator(space, w_obj):
+    return space.int(w_obj)
+
+def descr_get_denominator(space, w_obj):
+    return space.wrap(1)
+
+def descr_get_real(space, w_obj):
+    return space.int(w_obj)
+
+def descr_get_imag(space, w_obj):
+    return space.wrap(0)
 
 
 class W_IntObject(W_AbstractIntObject):
@@ -617,10 +632,15 @@
 
     conjugate = interpindirect2app(W_AbstractIntObject.descr_conjugate),
     bit_length = interpindirect2app(W_AbstractIntObject.descr_bit_length),
-    numerator = typedef.GetSetProperty(W_AbstractIntObject.descr_get_numerator),
-    denominator = typedef.GetSetProperty(W_AbstractIntObject.descr_get_denominator),
-    real = typedef.GetSetProperty(W_AbstractIntObject.descr_get_real),
-    imag = typedef.GetSetProperty(W_AbstractIntObject.descr_get_imag),
+    # XXX: need a GetSetIndirectProperty
+    #numerator = typedef.GetSetProperty(W_IntObject.descr_get_numerator),
+    #denominator = typedef.GetSetProperty(W_IntObject.descr_get_denominator),
+    #real = typedef.GetSetProperty(W_IntObject.descr_get_real),
+    #imag = typedef.GetSetProperty(W_IntObject.descr_get_imag),
+    numerator = typedef.GetSetProperty(descr_get_numerator),
+    denominator = typedef.GetSetProperty(descr_get_denominator),
+    real = typedef.GetSetProperty(descr_get_real),
+    imag = typedef.GetSetProperty(descr_get_imag),
     __int__ = interpindirect2app(W_AbstractIntObject.int),
     __long__ = interpindirect2app(W_AbstractIntObject.descr_long),
 

From noreply at buildbot.pypy.org  Thu Oct  3 07:53:28 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  3 Oct 2013 07:53:28 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: finish the talk
Message-ID: <20131003055328.DE7BA1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5065:d5a8d289edd6
Date: 2013-10-03 07:53 +0200
http://bitbucket.org/pypy/extradoc/changeset/d5a8d289edd6/

Log:	finish the talk

diff --git a/talk/fscons2012/talk.pdf b/talk/fscons2012/talk.pdf
index 1a6257d12b0ddced11a617033ddb00dbafc7612d..53828b62beb908f0b3f8ab8d72815e376c744719
GIT binary patch

[cut]

diff --git a/talk/pyconza2013/Makefile b/talk/pyconza2013/Makefile
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2013/Makefile
@@ -0,0 +1,13 @@
+
+
+view: talk.pdf
+	xpdf talk.pdf
+
+talk.pdf: talk.tex
+	64bit pdflatex talk.tex
+
+talk.tex: talk1.tex fix.py
+	python fix.py < talk1.tex > talk.tex
+
+talk1.tex: talk.rst
+	rst2beamer $< > talk1.tex
diff --git a/talk/pycon2013/pypy_without_gil/beamerdefs.txt b/talk/pyconza2013/beamerdefs.txt
copy from talk/pycon2013/pypy_without_gil/beamerdefs.txt
copy to talk/pyconza2013/beamerdefs.txt
diff --git a/talk/pyconza2013/fix.py b/talk/pyconza2013/fix.py
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2013/fix.py
@@ -0,0 +1,8 @@
+import sys
+
+for line in sys.stdin:
+    #if line.startswith('\\begin{itemize}'):
+    #    line = '\\begin{itemize}\n'
+    if line == '\\usepackage[scaled=.90]{helvet}\n':
+        line = '\\usepackage[scaled=1.1]{helvet}'
+    sys.stdout.write(line)
diff --git a/talk/pyconza2013/speed.png b/talk/pyconza2013/speed.png
new file mode 100644
index 0000000000000000000000000000000000000000..a99b2b6eb60e6c3df3118df660396286c98a41e0
GIT binary patch

[cut]

diff --git a/talk/pyconza2013/talk.pdf b/talk/pyconza2013/talk.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..6fed83a5c845e1d71cd4c32a98eb6a6b93d07bcf
GIT binary patch

[cut]

diff --git a/talk/pyconza2013/talk.rst b/talk/pyconza2013/talk.rst
--- a/talk/pyconza2013/talk.rst
+++ b/talk/pyconza2013/talk.rst
@@ -1,9 +1,20 @@
+.. include:: beamerdefs.txt
 
 =======================================
 Software Transactional Memory with PyPy
 =======================================
 
 
+Software Transactional Memory with PyPy
+---------------------------------------
+
+* PyCon ZA 2013
+
+* talk by Armin Rigo
+
+* sponsored by crowdfunding (thanks!)
+
+
 Introduction
 ------------
 
@@ -15,11 +26,13 @@
 Introduction
 ------------
 
-.. image: speed.png
+.. image:: speed.png
+   :scale: 65%
+   :align: center
 
 
-SQL Databases by example
-------------------------
+SQL by example
+--------------
 
 ::
 
@@ -34,10 +47,10 @@
 
 ::
 
-    ..
+    ...
     x = obj.value
     obj.value = x + 1
-    ..
+    ...
 
 
 Python by example
@@ -56,9 +69,10 @@
 
 ::
 
-    with atomic:
-        x = obj.value
-        obj.value = x + 1
+    the_lock.acquire()
+    x = obj.value
+    obj.value = x + 1
+    the_lock.release()
 
 
 Python by example
@@ -71,6 +85,16 @@
         obj.value = x + 1
 
 
+Python by example
+-----------------
+
+::
+
+    with atomic:
+        x = obj.value
+        obj.value = x + 1
+
+
 Locks != Transactions
 ---------------------
 
@@ -107,7 +131,7 @@
 
 * Transactional Memory
 
-* advanced magic (but not more so than databases)
+* advanced but not magic (same as databases)
 
 
 STM versus HTM
@@ -152,6 +176,8 @@
 
 * each thread runs tasks in a `with atomic`
 
+* uses threads, but internally only
+
 
 Example 2
 ---------
@@ -172,15 +198,36 @@
 
 ::
 
-  def compute(train):
+  def compute_time(train):
      ...
+     train.start_time = ...
 
   def next_iteration(all_trains):
      for train in all_trains:
-        add_task(compute, train)
+        add_task(compute_time, train)
      run_all_tasks()
 
 
+Conflicts
+---------
+
+* like database transactions
+
+* but with `objects` instead of `records`
+
+* the transaction aborts and automatically retries
+
+
+Inevitable
+----------
+
+* means "unavoidable"
+
+* handles I/O in a `with atomic`
+
+* cannot abort the transaction any more
+
+
 By the way
 ----------
 
@@ -192,29 +239,110 @@
 Current status
 --------------
 
-* 
+* basics work, JIT compiler integration almost done
 
+* different executable called `pypy-stm`
+
+* slow-down: around 3x (in bad cases up to 10x)
+
+* speed-ups measured with 4 cores
+
+* Linux 64-bit only
 
 
 User feedback
 -------------
 
+* implemented:
+
 ::
 
-  Detected conflict:
-    File "foo.py", line 17, in walk
-      if node.left not in seen:
-  Transaction aborted, 0.000047 seconds lost
+    Detected conflict:
+      File "foo.py", line 17, in walk
+        if node.left not in seen:
+    Transaction aborted, 0.000047 seconds lost
 
 
 User feedback
 -------------
 
+* not implemented yet:
+
 ::
 
-  Forced inevitable:
-    File "foo.py", line 19, in walk
-      print >> log, logentry
-  Transaction blocked others for 0.xx seconds
+    Forced inevitable:
+      File "foo.py", line 19, in walk
+        print >> log, logentry
+    Transaction blocked others for 0.xx seconds
 
-(not implemented yet)
+
+Async libraries
+---------------
+
+* future work
+
+* tweak a Twisted reactor: run multithreaded,
+  but use `with atomic`
+
+* existing Twisted apps still work, but we need to
+  look at conflicts/inevitables
+
+* similar with Tornado, gevent, and so on
+
+
+Async libraries
+---------------
+
+::
+
+  while True:
+     events = epoll.poll()
+     for event in events:
+        queue.put(event)
+
+And in several threads::
+
+  while True:
+     event = queue.get()
+     with atomic:
+        handle(event)
+
+
+More future work
+----------------
+
+* look at many more examples
+
+* tweak data structures to avoid conflicts
+
+* reduce slow-down, port to other OS'es
+
+
+Under the cover
+---------------
+
+* 10'000-feet overview
+
+* every object can have multiple versions
+
+* the shared versions are immutable
+
+* the most recent version can belong to one thread
+
+* synchronization only when a thread "steals" another thread's most
+  recent version, to make it shared
+
+* integrated with a generational garbage collector, with one
+  nursery per thread
+
+
+Summary
+-------
+
+* transactions in Python
+
+* a big change under the cover
+
+* a small change for Python users
+
+* `Q & A`

From noreply at buildbot.pypy.org  Fri Oct  4 07:04:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  4 Oct 2013 07:04:38 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Add 'with stm_ignored:' to not track
 in stm the reads and writes
Message-ID: <20131004050438.74E861C014D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r67146:a692bbaba9aa
Date: 2013-10-04 05:49 +0200
http://bitbucket.org/pypy/pypy/changeset/a692bbaba9aa/

Log:	Add 'with stm_ignored:' to not track in stm the reads and writes to
	GC objects

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -780,3 +780,37 @@
             raise Exception("import_from_mixin: would overwrite the value "
                             "already defined locally for %r" % (key,))
         target[key] = value
+
+# ____________________________________________________________
+
+class _StmIgnored:
+    def __enter__(self):
+        "NOT_RPYTHON"
+    def __exit__(self, *args):
+        "NOT_RPYTHON"
+
+class Entry(ExtRegistryEntry):
+    _about_ = _StmIgnored.__enter__.im_func
+    def compute_result_annotation(self, *args_s):
+        return None
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        hop.genop('stm_ignored_start', [])
+
+class Entry(ExtRegistryEntry):
+    _about_ = _StmIgnored.__exit__.im_func
+    def compute_result_annotation(self, *args_s):
+        return None
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        hop.genop('stm_ignored_stop', [])
+
+# Use "with stm_ignored:" around simple field read/write operations
+# that should not be tracked by the STM machinery.  They are always
+# simply performed instead.  It is useful for read/writes that don't
+# need to give a really consistent operation, when an approximative
+# behavior is fine, like incrementing some global counter.
+# XXX only for GC objects for now
+# XXX but it should replace 'stm_dont_track_raw_accesses' too
+# XXX DON'T USE for *writes* of a GC pointer into an object
+stm_ignored = _StmIgnored()
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -449,6 +449,9 @@
     'stm_get_adr_of_private_rev_num':LLOp(),
     'stm_get_adr_of_read_barrier_cache':LLOp(),
 
+    'stm_ignored_start':      LLOp(canrun=True),
+    'stm_ignored_stop':       LLOp(canrun=True),
+
     # __________ address operations __________
 
     'boehm_malloc':         LLOp(),
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -672,6 +672,12 @@
 def op_debug_stm_flush_barrier():
     pass
 
+def op_stm_ignored_start():
+    pass
+
+def op_stm_ignored_stop():
+    pass
+
 def op_stm_ptr_eq(x, y):
     return op_ptr_eq(x, y)
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -618,6 +618,12 @@
     OP_STM_CLEAR_EXCEPTION_DATA_ON_ABORT= _OP_STM
     OP_STM_ALLOCATE_NONMOVABLE_INT_ADR  = _OP_STM
 
+    def OP_STM_IGNORED_START(self, op):
+        return '/* stm_ignored_start */'
+
+    def OP_STM_IGNORED_STOP(self, op):
+        return '/* stm_ignored_stop */'
+
 
     def OP_PTR_NONZERO(self, op):
         return '%s = (%s != NULL);' % (self.expr(op.result),
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -918,6 +918,16 @@
         self.compile(entry_point)
         # assert did not explode
 
+    def test_ignore_stm_ignored(self):
+        from rpython.rlib.objectmodel import stm_ignored
+        def entry_point(argv):
+            with stm_ignored:
+                return len(argv)
+
+        self.compile(entry_point)
+        # assert did not explode
+
+
 class TestMaemo(TestStandalone):
     def setup_class(cls):
         py.test.skip("TestMaemo: tests skipped for now")
diff --git a/rpython/translator/stm/test/test_writebarrier.py b/rpython/translator/stm/test/test_writebarrier.py
--- a/rpython/translator/stm/test/test_writebarrier.py
+++ b/rpython/translator/stm/test/test_writebarrier.py
@@ -530,6 +530,31 @@
         self.interpret(f1, [])
         assert self.barriers == ['I2W']
 
+    def test_stm_ignored_1(self):
+        from rpython.rlib.objectmodel import stm_ignored
+        class Foo:
+            bar = 0
+        x = Foo()
+        def f1():
+            with stm_ignored:
+                x.bar += 2
+
+        self.interpret(f1, [])
+        assert self.barriers == []
+
+    def test_stm_ignored_2(self):
+        from rpython.rlib.objectmodel import stm_ignored
+        class Foo:
+            bar = 0
+        def f1():
+            y = Foo()
+            llop.debug_stm_flush_barrier(lltype.Void)
+            with stm_ignored:
+                y.bar += 2
+
+        self.interpret(f1, [])
+        assert self.barriers == ['a2i']
+
 
 external_release_gil = rffi.llexternal('external_release_gil', [], lltype.Void,
                                        _callable=lambda: None,
diff --git a/rpython/translator/stm/test/transform_support.py b/rpython/translator/stm/test/transform_support.py
--- a/rpython/translator/stm/test/transform_support.py
+++ b/rpython/translator/stm/test/transform_support.py
@@ -66,6 +66,7 @@
 
 
 class LLSTMFrame(LLFrame):
+    stm_ignored = False
 
     def all_stm_ptrs(self):
         for frame in self.llinterpreter.frame_stack:
@@ -80,6 +81,11 @@
         cat = self.get_category_or_null(p)
         assert cat is None or cat in 'AIQRVW'
         if expected is not None:
+            if self.stm_ignored:
+                if expected >= 'W':
+                    raise AssertionError("should not be seen in 'stm_ignored'")
+                if expected > 'I':
+                    expected = 'I'
             assert cat is not None and cat >= expected
         return cat
 
@@ -99,6 +105,14 @@
             self.llinterpreter.tester.barriers.append(kind)
             return ptr2
 
+    def op_stm_ignored_start(self):
+        assert self.stm_ignored == False
+        self.stm_ignored = True
+
+    def op_stm_ignored_stop(self):
+        assert self.stm_ignored == True
+        self.stm_ignored = False
+
     def op_stm_ptr_eq(self, obj1, obj2):
         self.check_category(obj1, None)
         self.check_category(obj2, None)
diff --git a/rpython/translator/stm/writebarrier.py b/rpython/translator/stm/writebarrier.py
--- a/rpython/translator/stm/writebarrier.py
+++ b/rpython/translator/stm/writebarrier.py
@@ -3,6 +3,7 @@
 from rpython.translator.unsimplify import insert_empty_startblock
 from rpython.rtyper.lltypesystem import lltype
 from rpython.translator.backendopt.writeanalyze import top_set
+from rpython.translator.simplify import join_blocks
 
 
 MALLOCS = set([
@@ -65,11 +66,12 @@
         self.update_inputargs_category()
 
 
-    def analyze_inside_block(self):
+    def analyze_inside_block(self, graph):
         gcremovetypeptr = (
             self.stmtransformer.translator.config.translation.gcremovetypeptr)
         wants_a_barrier = {}
         expand_comparison = set()
+        stm_ignored = False
         for op in self.block.operations:
             is_getter = (op.opname in ('getfield', 'getarrayitem',
                                        'getinteriorfield', 'raw_load') and
@@ -118,7 +120,26 @@
 
             elif op.opname == 'gc_writebarrier':
                 wants_a_barrier[op] = 'W'
+
+            elif op.opname == 'stm_ignored_start':
+                assert not stm_ignored, "nested 'with stm_ignored'"
+                stm_ignored = True
+
+            elif op.opname == 'stm_ignored_stop':
+                assert stm_ignored, "stm_ignored_stop without start?"
+                stm_ignored = False
+
+            if stm_ignored and op in wants_a_barrier:
+                if wants_a_barrier[op] == 'W':
+                    raise Exception(
+                        "%r: 'with stm_ignored:' contains unsupported "
+                        "operation %r writing a GC pointer" % (graph, op))
+                assert 'I' <= wants_a_barrier[op] < 'W'
+                wants_a_barrier[op] = 'I'
         #
+        if stm_ignored:
+            raise Exception("%r: 'with stm_ignored:' code body too complex"
+                            % (graph,))
         self.wants_a_barrier = wants_a_barrier
         self.expand_comparison = expand_comparison
 
@@ -356,6 +377,7 @@
        The letters are chosen so that a barrier is needed to change a
        pointer from category x to category y if and only if y > x.
     """
+    join_blocks(graph)
     graphinfo = stmtransformer.write_analyzer.compute_graph_info(graph)
     annotator = stmtransformer.translator.annotator
     insert_empty_startblock(annotator, graph)
@@ -366,7 +388,7 @@
         if block.operations == ():
             continue
         bt = BlockTransformer(stmtransformer, block)
-        bt.analyze_inside_block()
+        bt.analyze_inside_block(graph)
         block_transformers[block] = bt
 
     bt = block_transformers[graph.startblock]

From noreply at buildbot.pypy.org  Fri Oct  4 07:04:39 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  4 Oct 2013 07:04:39 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Use stm_ignored here
Message-ID: <20131004050439.AD1071C01B0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r67147:e6fe5ef81974
Date: 2013-10-04 05:50 +0200
http://bitbucket.org/pypy/pypy/changeset/e6fe5ef81974/

Log:	Use stm_ignored here

diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -4,7 +4,7 @@
 from rpython.rlib import jit, types, rgc
 from rpython.rlib.debug import ll_assert
 from rpython.rlib.objectmodel import (malloc_zero_filled, we_are_translated,
-    _hash_string, keepalive_until_here, specialize)
+    _hash_string, keepalive_until_here, specialize, stm_ignored)
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.error import TyperError
@@ -342,12 +342,14 @@
         # special non-computed-yet value.
         if not s:
             return 0
-        x = s.hash
+        with stm_ignored:
+            x = s.hash
         if x == 0:
             x = _hash_string(s.chars)
             if x == 0:
                 x = 29872897
-            s.hash = x
+            with stm_ignored:
+                s.hash = x
         return x
 
     def ll_length(s):

From noreply at buildbot.pypy.org  Fri Oct  4 07:04:44 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  4 Oct 2013 07:04:44 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Use stm_ignored on the guard counters
	too.
Message-ID: <20131004050444.803EA1DAE39@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r67151:649185c19ab4
Date: 2013-10-04 07:03 +0200
http://bitbucket.org/pypy/pypy/changeset/649185c19ab4/

Log:	Use stm_ignored on the guard counters too.

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1,9 +1,9 @@
 import weakref
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, stm_ignored
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
-from rpython.rlib import rstack
+from rpython.rlib import rstack, rgc
 from rpython.rlib.jit import JitDebugInfo, Counters, dont_look_inside
 from rpython.conftest import option
 from rpython.tool.sourcetools import func_with_new_name
@@ -512,6 +512,8 @@
         self.guard_opnum = guard_op.getopnum()
 
     def make_a_counter_per_value(self, guard_value_op):
+        if rgc.stm_is_enabled():
+            return    # XXX don't use the special counters in stm mode for now
         assert guard_value_op.getopnum() == rop.GUARD_VALUE
         box = guard_value_op.getarg(0)
         try:
@@ -557,21 +559,35 @@
     _trace_and_compile_from_bridge._dont_inline_ = True
 
     def must_compile(self, deadframe, metainterp_sd, jitdriver_sd):
+        ok = self.must_compile_approx(deadframe, metainterp_sd, jitdriver_sd)
+        if not rgc.stm_is_enabled():
+            return ok
+        else:
+            # in stm mode, the return value may (rarely) be True even if a
+            # real, stm-protected read of self._counter says "busy".
+            return ok and not (self._counter & self.CNT_BUSY_FLAG)
+
+    def must_compile_approx(self, deadframe, metainterp_sd, jitdriver_sd):
         trace_eagerness = jitdriver_sd.warmstate.trace_eagerness
         #
-        if self._counter <= self.CNT_BASE_MASK:
+        with stm_ignored:
+            approx_counter = self._counter
+        if approx_counter <= self.CNT_BASE_MASK:
             # simple case: just counting from 0 to trace_eagerness
-            self._counter += 1
-            return self._counter >= trace_eagerness
+            approx_counter += 1
+            with stm_ignored:
+                self._counter = approx_counter
+            return approx_counter >= trace_eagerness
         #
         # do we have the BUSY flag?  If so, we're tracing right now, e.g. in an
         # outer invocation of the same function, so don't trace again for now.
-        elif self._counter & self.CNT_BUSY_FLAG:
+        elif approx_counter & self.CNT_BUSY_FLAG:
             return False
         #
         else: # we have a GUARD_VALUE that fails.  Make a _counters instance
             # (only now, when the guard is actually failing at least once),
             # and use it to record some statistics about the failing values.
+            assert not rgc.stm_is_enabled(), "XXX"
             index = self._counter & self.CNT_BASE_MASK
             typetag = self._counter & self.CNT_TYPE_MASK
             counters = self._counters

From noreply at buildbot.pypy.org  Fri Oct  4 07:04:40 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  4 Oct 2013 07:04:40 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Fix for e6fe5ef81974
Message-ID: <20131004050440.D815A1C02A7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r67148:622d65b97854
Date: 2013-10-04 06:01 +0200
http://bitbucket.org/pypy/pypy/changeset/622d65b97854/

Log:	Fix for e6fe5ef81974

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -788,22 +788,8 @@
         "NOT_RPYTHON"
     def __exit__(self, *args):
         "NOT_RPYTHON"
-
-class Entry(ExtRegistryEntry):
-    _about_ = _StmIgnored.__enter__.im_func
-    def compute_result_annotation(self, *args_s):
-        return None
-    def specialize_call(self, hop):
-        hop.exception_cannot_occur()
-        hop.genop('stm_ignored_start', [])
-
-class Entry(ExtRegistryEntry):
-    _about_ = _StmIgnored.__exit__.im_func
-    def compute_result_annotation(self, *args_s):
-        return None
-    def specialize_call(self, hop):
-        hop.exception_cannot_occur()
-        hop.genop('stm_ignored_stop', [])
+    def _freeze_(self):
+        return True
 
 # Use "with stm_ignored:" around simple field read/write operations
 # that should not be tracked by the STM machinery.  They are always
@@ -814,3 +800,24 @@
 # XXX but it should replace 'stm_dont_track_raw_accesses' too
 # XXX DON'T USE for *writes* of a GC pointer into an object
 stm_ignored = _StmIgnored()
+
+
+# RPython hacks
+def _stm_ignored_start(): "NOT_RPYTHON"
+def _stm_ignored_stop(exc, val, tb): "NOT_RPYTHON"
+stm_ignored.__enter__ = _stm_ignored_start
+stm_ignored.__exit__ = _stm_ignored_stop
+class Entry(ExtRegistryEntry):
+    _about_ = _stm_ignored_start
+    def compute_result_annotation(self, *args_s):
+        return None
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        hop.genop('stm_ignored_start', [])
+class Entry(ExtRegistryEntry):
+    _about_ = _stm_ignored_stop
+    def compute_result_annotation(self, *args_s):
+        return None
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        hop.genop('stm_ignored_stop', [])

From noreply at buildbot.pypy.org  Fri Oct  4 07:04:42 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  4 Oct 2013 07:04:42 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Should carefully have no effect: split
 the double-meaning of the 'counter'
Message-ID: <20131004050442.077711C11B6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r67149:0fe3cbf3f18d
Date: 2013-10-04 06:40 +0200
http://bitbucket.org/pypy/pypy/changeset/0fe3cbf3f18d/

Log:	Should carefully have no effect: split the double-meaning of the
	'counter' on Cell objects in two fields.

diff --git a/rpython/jit/metainterp/test/test_warmstate.py b/rpython/jit/metainterp/test/test_warmstate.py
--- a/rpython/jit/metainterp/test/test_warmstate.py
+++ b/rpython/jit/metainterp/test/test_warmstate.py
@@ -4,6 +4,7 @@
 from rpython.jit.metainterp.warmstate import wrap, unwrap, specialize_value
 from rpython.jit.metainterp.warmstate import equal_whatever, hash_whatever
 from rpython.jit.metainterp.warmstate import WarmEnterState, JitCell
+from rpython.jit.metainterp.warmstate import MODE_HAVE_PROC, MODE_TRACING
 from rpython.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from rpython.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr
 from rpython.jit.codewriter import longlong
@@ -162,7 +163,7 @@
                                       constfloat(2.25)],
                                      looptoken)
     cell1 = get_jitcell(True, 5, 2.25)
-    assert cell1.counter < 0
+    assert cell1.mode == MODE_HAVE_PROC
     assert cell1.get_procedure_token() is looptoken
 
 def test_make_jitdriver_callbacks_1():
@@ -299,17 +300,17 @@
     #
     for i in range(1, 20005):
         cell = get_jitcell(True, i)
-        cell.counter = -1
+        cell.mode = MODE_HAVE_PROC
         cell.wref_procedure_token = None    # or a dead weakref, equivalently
         assert len(warmstate._jitcell_dict) == (i % 20000) + 1
     #
-    # Same test, with counter == -2 (rare case, kept alive)
+    # Same test, with mode == MODE_TRACING (rare case, kept alive)
     warmstate = WarmEnterState(None, FakeJitDriverSD())
     get_jitcell = warmstate._make_jitcell_getter_default()
     cell = get_jitcell(True, -1)
-    cell.counter = -2
+    cell.mode = MODE_TRACING
     #
     for i in range(1, 20005):
         cell = get_jitcell(True, i)
-        cell.counter = -2
+        cell.mode = MODE_TRACING
         assert len(warmstate._jitcell_dict) == i + 1
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -124,12 +124,13 @@
         return rffi.cast(lltype.Signed, x)
 
 
+MODE_COUNTING  = '\x00'   # not yet traced, wait till threshold is reached
+MODE_TRACING   = 'T'      # tracing is currently going on for this cell
+MODE_HAVE_PROC = 'P'      # there is an entry bridge for this cell
+
 class JitCell(BaseJitCell):
-    # the counter can mean the following things:
-    #     counter >=  0: not yet traced, wait till threshold is reached
-    #     counter == -1: there is an entry bridge for this cell
-    #     counter == -2: tracing is currently going on for this cell
-    counter = 0
+    counter = 0    # when THRESHOLD_LIMIT is reached, start tracing
+    mode = MODE_COUNTING
     dont_trace_here = False
     extra_delay = chr(0)
     wref_procedure_token = None
@@ -241,7 +242,7 @@
         cell = self.jit_cell_at_key(greenkey)
         old_token = cell.get_procedure_token()
         cell.set_procedure_token(procedure_token)
-        cell.counter = -1       # valid procedure bridge attached
+        cell.mode = MODE_HAVE_PROC       # valid procedure bridge attached
         if old_token is not None:
             self.cpu.redirect_call_assembler(old_token, procedure_token)
             # procedure_token is also kept alive by any loop that used
@@ -320,19 +321,19 @@
                 cell.extra_delay = curgen
                 return
             #
+            cell.counter = 0
             if not confirm_enter_jit(*args):
-                cell.counter = 0
                 return
             # start tracing
             from rpython.jit.metainterp.pyjitpl import MetaInterp
             metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
-            # set counter to -2, to mean "tracing in effect"
-            cell.counter = -2
+            cell.mode = MODE_TRACING
             try:
                 metainterp.compile_and_run_once(jitdriver_sd, *args)
             finally:
-                if cell.counter == -2:
+                if cell.mode == MODE_TRACING:
                     cell.counter = 0
+                    cell.mode = MODE_COUNTING
 
         def maybe_compile_and_run(threshold, *args):
             """Entry point to the JIT.  Called at the point with the
@@ -341,8 +342,9 @@
             # look for the cell corresponding to the current greenargs
             greenargs = args[:num_green_args]
             cell = get_jitcell(True, *greenargs)
+            mode = cell.mode
 
-            if cell.counter >= 0:
+            if mode == MODE_COUNTING:
                 # update the profiling counter
                 n = cell.counter + threshold
                 if n <= self.THRESHOLD_LIMIT:       # bound not reached
@@ -351,9 +353,10 @@
                 else:
                     bound_reached(cell, *args)
                     return
+
             else:
-                if cell.counter != -1:
-                    assert cell.counter == -2
+                if mode != MODE_HAVE_PROC:
+                    assert mode == MODE_TRACING
                     # tracing already happening in some outer invocation of
                     # this function. don't trace a second time.
                     return
@@ -363,6 +366,7 @@
                 procedure_token = cell.get_procedure_token()
                 if procedure_token is None:   # it was a weakref that has been freed
                     cell.counter = 0
+                    cell.mode = MODE_COUNTING
                     return
                 # extract and unspecialize the red arguments to pass to
                 # the assembler
@@ -459,11 +463,11 @@
             minimum = self.THRESHOLD_LIMIT // 20     # minimum 5%
             killme = []
             for key, cell in jitcell_dict.iteritems():
-                if cell.counter >= 0:
+                if cell.mode == MODE_COUNTING:
                     cell.counter = int(cell.counter * 0.92)
                     if cell.counter < minimum:
                         killme.append(key)
-                elif (cell.counter == -1
+                elif (cell.mode == MODE_HAVE_PROC
                       and cell.get_procedure_token() is None):
                     killme.append(key)
             for key in killme:
@@ -589,8 +593,11 @@
             procedure_token = cell.get_procedure_token()
             if procedure_token is None:
                 from rpython.jit.metainterp.compile import compile_tmp_callback
-                if cell.counter == -1:    # used to be a valid entry bridge,
-                    cell.counter = 0      # but was freed in the meantime.
+                if cell.mode == MODE_HAVE_PROC:
+                    # used to be a valid entry bridge,
+                    # but was freed in the meantime.
+                    cell.counter = 0
+                    cell.mode = MODE_COUNTING
                 memmgr = warmrunnerdesc.memory_manager
                 procedure_token = compile_tmp_callback(cpu, jd, greenkey,
                                                        redargtypes, memmgr)

From noreply at buildbot.pypy.org  Fri Oct  4 07:04:43 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  4 Oct 2013 07:04:43 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Apply stm_ignored to this version of
	cell counters
Message-ID: <20131004050443.59C231DAE38@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r67150:e208b4c38797
Date: 2013-10-04 06:42 +0200
http://bitbucket.org/pypy/pypy/changeset/e208b4c38797/

Log:	Apply stm_ignored to this version of cell counters

diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -7,6 +7,7 @@
 from rpython.rlib.jit import PARAMETERS, BaseJitCell
 from rpython.rlib.nonconst import NonConstant
 from rpython.rlib.objectmodel import specialize, we_are_translated, r_dict
+from rpython.rlib.objectmodel import stm_ignored
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.annlowlevel import (hlstr, cast_base_ptr_to_instance,
@@ -326,8 +327,8 @@
                 return
             # start tracing
             from rpython.jit.metainterp.pyjitpl import MetaInterp
+            cell.mode = MODE_TRACING
             metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
-            cell.mode = MODE_TRACING
             try:
                 metainterp.compile_and_run_once(jitdriver_sd, *args)
             finally:
@@ -346,9 +347,13 @@
 
             if mode == MODE_COUNTING:
                 # update the profiling counter
-                n = cell.counter + threshold
+                # use stm_ignored because keeping the absolutely exact value
+                # is not important, but avoiding pointless conflicts is
+                with stm_ignored:
+                    n = cell.counter + threshold
                 if n <= self.THRESHOLD_LIMIT:       # bound not reached
-                    cell.counter = n
+                    with stm_ignored:
+                        cell.counter = n
                     return
                 else:
                     bound_reached(cell, *args)

From noreply at buildbot.pypy.org  Fri Oct  4 10:08:07 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  4 Oct 2013 10:08:07 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: my talk
Message-ID: <20131004080807.37B171C0934@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5066:3a007c1729fb
Date: 2013-10-04 10:05 +0200
http://bitbucket.org/pypy/extradoc/changeset/3a007c1729fb/

Log:	my talk

diff --git a/talk/pyconza2013/php/Makefile b/talk/pyconza2013/php/Makefile
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2013/php/Makefile
@@ -0,0 +1,13 @@
+
+
+view: talk.pdf
+	evince talk.pdf
+
+talk.pdf: talk.tex
+	pdflatex talk.tex
+
+talk.tex: talk.rst
+	rst2beamer --stylesheet=stylesheet.latex --documentoptions=14pt --input-encoding=utf8 --output-encoding=utf8 --overlaybullets=false $< > talk.tex
+
+clean:
+	rm talk1.tex talk.tex talk.pdf
diff --git a/talk/pyconza2013/php/beamerdefs.txt b/talk/pyconza2013/php/beamerdefs.txt
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2013/php/beamerdefs.txt
@@ -0,0 +1,108 @@
+.. colors
+.. ===========================
+
+.. role:: green
+.. role:: red
+
+
+.. general useful commands
+.. ===========================
+
+.. |pause| raw:: latex
+
+   \pause
+
+.. |small| raw:: latex
+
+   {\small
+
+.. |end_small| raw:: latex
+
+   }
+
+.. |scriptsize| raw:: latex
+
+   {\scriptsize
+
+.. |end_scriptsize| raw:: latex
+
+   }
+
+.. |strike<| raw:: latex
+
+   \sout{
+
+.. closed bracket
+.. ===========================
+
+.. |>| raw:: latex
+
+   }
+
+
+.. example block
+.. ===========================
+
+.. |example<| raw:: latex
+
+   \begin{exampleblock}{
+
+
+.. |end_example| raw:: latex
+
+   \end{exampleblock}
+
+
+
+.. alert block
+.. ===========================
+
+.. |alert<| raw:: latex
+
+   \begin{alertblock}{
+
+
+.. |end_alert| raw:: latex
+
+   \end{alertblock}
+
+
+
+.. columns
+.. ===========================
+
+.. |column1| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.45\textwidth}
+
+.. |column2| raw:: latex
+
+      \end{column}
+      \begin{column}{0.45\textwidth}
+
+
+.. |end_columns| raw:: latex
+
+      \end{column}
+   \end{columns}
+
+
+
+.. |snake| image:: ../../img/py-web-new.png
+           :scale: 15%
+           
+
+
+.. nested blocks
+.. ===========================
+
+.. |nested| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.85\textwidth}
+
+.. |end_nested| raw:: latex
+
+      \end{column}
+   \end{columns}
diff --git a/talk/pyconza2013/php/stylesheet.latex b/talk/pyconza2013/php/stylesheet.latex
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2013/php/stylesheet.latex
@@ -0,0 +1,10 @@
+\usetheme{Warsaw}
+\usecolortheme{whale}
+\setbeamercovered{transparent}
+\definecolor{darkgreen}{rgb}{0, 0.5, 0.0}
+\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor}
+\addtobeamertemplate{block begin}{}{\setlength{\parskip}{35pt plus 1pt minus 1pt}}
+
+\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\red}[1]{\color{red}#1\normalcolor}
diff --git a/talk/pyconza2013/php/talk.pdf b/talk/pyconza2013/php/talk.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..c3455b8c36fbba1f91ef50062af1704ccdce201f
GIT binary patch

[cut]

diff --git a/talk/pyconza2013/php/talk.rst b/talk/pyconza2013/php/talk.rst
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2013/php/talk.rst
@@ -0,0 +1,160 @@
+
+.. include:: beamerdefs.txt
+
+.. raw:: latex
+
+   \title{PHP interpreter using PyPy technology}
+   \author[fijal]{Maciej Fija&#322;kowski}
+
+   \institute{PyCon ZA 2013}
+   \date{4th October 2013}
+
+   \maketitle
+
+introduction
+------------
+
+* me - Maciej Fija&#322;kowski, PyPy core developer
+
+* technology - PyPy
+
+* project - PHP interpreter
+
+Wait, what???!!!1
+-----------------
+
+* PHP - by far the most popular language on the web
+
+* PyPy - proven technology for speeding up Python
+
+* examples who can gain: wikipedia, facebook, wordpress....
+
+Current landscape
+-----------------
+
+* Zend - a simple, bytecode-based interpreter
+
+* HipHop - PHP to C++ compiler, facebook project
+
+* HHVM - successor to HipHop, JIT-based, also facebook
+
+Current benchmarks landscape
+----------------------------
+
+* benchmarks are hard
+
+* a set I've seen is mostly numeric or language shootout
+
+* not very representative
+
+* HipHop is 2-4x faster than Zend, HHVM 40% faster than hiphop
+
+* no real-world PHP benchmark suite (a la speed.pypy.org)
+
+PyPy
+----
+
+* fast interpreter for a python language
+
+* but also, a toolchain for constructing interpreters
+
+* comes with a just-in-time compiler
+
+More about PyPy
+---------------
+
+* implementation language of PyPy is RPython
+
+* RPython is a subset of Python
+
+* RPython can be compiled statically to C
+
+|pause|
+
+* ... but also can have just in time compiler generated for
+
+More about RPython
+------------------
+
+* a great language for writing interpreters
+
+|pause|
+
+* a horrible language with tons of tricks
+
+* great results in good enough time
+
+* http://tratt.net/laurie/blog/entries/fast_enough_vms_in_fast_enough_time
+
+Introducing hippy
+-----------------
+
+* PHP interpreter written in RPython
+
+* bug-to-bug compatible with Zend
+
+* interpreter + just in time compiler (for free)
+
+* preliminary study sponsored by facebook
+
+* good preliminary performance results
+
+PHP is hard
+-----------
+
+* crazy standard library
+
+|pause|
+
+* function calls by name
+
+* ``*args`` equivalent, ``apply`` equivalent, etc.
+
+* crazy reference semantics
+
+* copy-on-write and refcounting
+
+\.\.\. but getting it fast is easier
+------------------------------------
+
+* we **do** have a JIT as soon as we write an interpreter
+
+* PyPy has really good technology
+
+Basic construction
+------------------
+
+* interpreter loop
+
+* standard library
+
+|pause|
+
+* typical stuff
+
+Web server integration
+----------------------
+
+* PHP is very request based with throwing out all the data in between
+
+* we want to persist as much as possible
+
+* various optimizations possible
+
+We're hiring
+------------
+
+* want to work on something obscure and challenging?
+
+* with smart people?
+
+* talk to me or Armin
+
+Q&A
+---
+
+* any questions?
+
+* fijall at gmail
+
+* http://baroquesoftware.com

From noreply at buildbot.pypy.org  Fri Oct  4 10:09:20 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  4 Oct 2013 10:09:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: make my emacs happier
Message-ID: <20131004080920.D00C71C0112@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67152:2cb50d52f649
Date: 2013-10-04 10:08 +0200
http://bitbucket.org/pypy/pypy/changeset/2cb50d52f649/

Log:	make my emacs happier

diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -197,10 +197,10 @@
 
 W_AbstractTupleObject.typedef = StdTypeDef(
     "tuple",
-    __doc__ = '''tuple() -> an empty tuple
+    __doc__ = """tuple() -> an empty tuple
 tuple(sequence) -> tuple initialized from sequence's items
 
-If the argument is a tuple, the return value is the same object.''',
+If the argument is a tuple, the return value is the same object.""",
     __new__ = interp2app(W_AbstractTupleObject.descr_new),
     __repr__ = interp2app(W_AbstractTupleObject.descr_repr),
     __hash__ = interpindirect2app(W_AbstractTupleObject.descr_hash),

From noreply at buildbot.pypy.org  Fri Oct  4 11:08:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  4 Oct 2013 11:08:41 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Tweak slides
Message-ID: <20131004090841.D0ADF1C13FB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5067:f7e98dc6084b
Date: 2013-10-04 11:01 +0200
http://bitbucket.org/pypy/extradoc/changeset/f7e98dc6084b/

Log:	Tweak slides

diff --git a/talk/pyconza2013/Makefile b/talk/pyconza2013/Makefile
--- a/talk/pyconza2013/Makefile
+++ b/talk/pyconza2013/Makefile
@@ -1,13 +1,13 @@
 
 
 view: talk.pdf
-	xpdf talk.pdf
+	evince talk.pdf
 
 talk.pdf: talk.tex
 	64bit pdflatex talk.tex
 
-talk.tex: talk1.tex fix.py
-	python fix.py < talk1.tex > talk.tex
+talk.tex: talk.rst
+	rst2beamer --stylesheet=stylesheet.latex --documentoptions=14pt --input-encoding=utf8 --output-encoding=utf8 --overlaybullets=false $< > talk.tex
 
-talk1.tex: talk.rst
-	rst2beamer $< > talk1.tex
+clean:
+	rm -f talk.tex talk.pdf
diff --git a/talk/pyconza2013/stylesheet.latex b/talk/pyconza2013/stylesheet.latex
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2013/stylesheet.latex
@@ -0,0 +1,10 @@
+\usetheme{Warsaw}
+\usecolortheme{whale}
+\setbeamercovered{transparent}
+\definecolor{darkgreen}{rgb}{0, 0.5, 0.0}
+\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor}
+\addtobeamertemplate{block begin}{}{\setlength{\parskip}{35pt plus 1pt minus 1pt}}
+
+\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\red}[1]{\color{red}#1\normalcolor}
diff --git a/talk/pyconza2013/talk.pdf b/talk/pyconza2013/talk.pdf
index 6fed83a5c845e1d71cd4c32a98eb6a6b93d07bcf..80a179ffd04df370eb9f786822977148c46dc6be
GIT binary patch

[cut]

diff --git a/talk/pyconza2013/talk.rst b/talk/pyconza2013/talk.rst
--- a/talk/pyconza2013/talk.rst
+++ b/talk/pyconza2013/talk.rst
@@ -1,18 +1,14 @@
 .. include:: beamerdefs.txt
 
-=======================================
-Software Transactional Memory with PyPy
-=======================================
+.. raw:: latex
 
+   \title{Software Transactional Memory with PyPy}
+   \author[arigo]{Armin Rigo}
 
-Software Transactional Memory with PyPy
----------------------------------------
+   \institute{PyCon ZA 2013}
+   \date{4th October 2013}
 
-* PyCon ZA 2013
-
-* talk by Armin Rigo
-
-* sponsored by crowdfunding (thanks!)
+   \maketitle
 
 
 Introduction
@@ -20,6 +16,8 @@
 
 * what is PyPy: an alternative implementation of Python
 
+* very compatible
+
 * main focus is on speed
 
 
@@ -34,6 +32,14 @@
 SQL by example
 --------------
 
+.. raw:: latex
+
+   %empty
+
+
+SQL by example
+--------------
+
 ::
 
     BEGIN TRANSACTION;
@@ -58,6 +64,27 @@
 
 ::
 
+    ...
+    obj.value += 1
+    ...
+
+
+Python by example
+-----------------
+
+::
+
+    ...
+    x = obj.value
+    obj.value = x + 1
+    ...
+
+
+Python by example
+-----------------
+
+::
+
     begin_transaction()
     x = obj.value
     obj.value = x + 1
@@ -100,10 +127,10 @@
 
 ::
 
-    BEGIN TRANSACTION;    BEGIN TRANSACTION;    BEGIN..
-    SELECT * FROM ...;    SELECT * FROM ...;    SELEC..
-    UPDATE ...;           UPDATE ...;           UPDAT..
-    COMMIT;               COMMIT;               COMMI..
+    BEGIN TRANSACTION;  BEGIN TRANSACTION;    BEGIN..
+    SELECT * FROM ...;  SELECT * FROM ...;    SELEC..
+    UPDATE ...;         UPDATE ...;           UPDAT..
+    COMMIT;             COMMIT;               COMMI..
 
 
 Locks != Transactions
@@ -111,9 +138,9 @@
 
 ::
 
-    with the_lock:        with the_lock:        with ..
-      x = obj.val           x = obj.val           x =..
-      obj.val = x + 1       obj.val = x + 1       obj..
+    with the_lock:      with the_lock:        with ..
+      x = obj.val         x = obj.val           x =..
+      obj.val = x + 1     obj.val = x + 1       obj..
 
 
 Locks != Transactions
@@ -121,9 +148,9 @@
 
 ::
 
-    with atomic:          with atomic:          with ..
-      x = obj.val           x = obj.val           x =..
-      obj.val = x + 1       obj.val = x + 1       obj..
+    with atomic:        with atomic:          with ..
+      x = obj.val         x = obj.val           x =..
+      obj.val = x + 1     obj.val = x + 1       obj..
 
 
 STM
@@ -134,14 +161,46 @@
 * advanced but not magic (same as databases)
 
 
-STM versus HTM
---------------
+By the way
+----------
 
-* Software versus Hardware
+* STM replaces the GIL (Global Interpreter Lock)
 
-* CPU hardware specially to avoid the high overhead
+* any existing multithreaded program runs on multiple cores
 
-* too limited for now
+
+By the way
+----------
+
+* the GIL is necessary and very hard to avoid,
+  but if you look at it like a lock around every single
+  subexpression, then it can be replaced with `with atomic` too
+
+
+So...
+-----
+
+* yes, any existing multithreaded program runs on multiple cores
+
+* yes, we solved the GIL
+
+* great
+
+
+So...
+-----
+
+* no, it would be quite hard to implement it in standard CPython
+
+* but not completely impossible
+
+* too bad for now, only in PyPy
+
+
+But...
+------
+
+* but only half of the story in my opinion `:-)`
 
 
 Example 1
@@ -149,11 +208,13 @@
 
 ::
 
-  def apply_interest_rate(self):
+  def apply_interest(self):
      self.balance *= 1.05
 
+
   for account in all_accounts:
-     account.apply_interest_rate()
+     account.apply_interest()
+                                                 .
 
 
 Example 1
@@ -161,11 +222,26 @@
 
 ::
 
-  def apply_interest_rate(self):
+  def apply_interest(self):
      self.balance *= 1.05
 
+
   for account in all_accounts:
-     add_task(account.apply_interest_rate)
+     account.apply_interest()
+     ^^^ run this loop multithreaded
+
+
+Example 1
+---------
+
+::
+
+  def apply_interest(self):
+     #with atomic: --- automatic
+        self.balance *= 1.05
+
+  for account in all_accounts:
+     add_task(account.apply_interest)
   run_tasks()
 
 
@@ -178,6 +254,8 @@
 
 * uses threads, but internally only
 
+* very simple, pure Python
+
 
 Example 2
 ---------
@@ -187,7 +265,7 @@
   def next_iteration(all_trains):
      for train in all_trains:
         start_time = ...
-        for othertrain in train.dependencies:
+        for othertrain in train.deps:
            if ...:
               start_time = ...
         train.start_time = start_time
@@ -215,37 +293,29 @@
 
 * but with `objects` instead of `records`
 
-* the transaction aborts and automatically retries
+* the transaction aborts and retries automatically
 
 
 Inevitable
 ----------
 
-* means "unavoidable"
+* "inevitable" (means "unavoidable")
 
 * handles I/O in a `with atomic`
 
 * cannot abort the transaction any more
 
 
-By the way
-----------
-
-* STM replaces the GIL
-
-* any existing multithreaded program runs on multiple cores
-
-
 Current status
 --------------
 
 * basics work, JIT compiler integration almost done
 
-* different executable called `pypy-stm`
+* different executable (`pypy-stm` instead of `pypy`)
 
 * slow-down: around 3x (in bad cases up to 10x)
 
-* speed-ups measured with 4 cores
+* real time speed-ups measured with 4 or 8 cores
 
 * Linux 64-bit only
 
@@ -258,9 +328,11 @@
 ::
 
     Detected conflict:
+      File "foo.py", line 58, in wtree
+        walk(root)
       File "foo.py", line 17, in walk
         if node.left not in seen:
-    Transaction aborted, 0.000047 seconds lost
+    Transaction aborted, 0.047 sec lost
 
 
 User feedback
@@ -273,11 +345,11 @@
     Forced inevitable:
       File "foo.py", line 19, in walk
         print >> log, logentry
-    Transaction blocked others for 0.xx seconds
+    Transaction blocked others for XX s
 
 
-Async libraries
----------------
+Asynchronous libraries
+----------------------
 
 * future work
 
@@ -287,11 +359,11 @@
 * existing Twisted apps still work, but we need to
   look at conflicts/inevitables
 
-* similar with Tornado, gevent, and so on
+* similar with Tornado, eventlib, and so on
 
 
-Async libraries
----------------
+Asynchronous libraries
+----------------------
 
 ::
 
@@ -318,6 +390,16 @@
 * reduce slow-down, port to other OS'es
 
 
+STM versus HTM
+--------------
+
+* Software versus Hardware
+
+* CPU hardware specially to avoid the high overhead (Intel Haswell processor)
+
+* too limited for now
+
+
 Under the cover
 ---------------
 
@@ -329,8 +411,8 @@
 
 * the most recent version can belong to one thread
 
-* synchronization only when a thread "steals" another thread's most
-  recent version, to make it shared
+* synchronization only at the point where one thread "steals"
+  another thread's most recent version, to make it shared
 
 * integrated with a generational garbage collector, with one
   nursery per thread
@@ -345,4 +427,8 @@
 
 * a small change for Python users
 
+* (and the GIL is gone)
+
+* this work is sponsored by crownfunding (thanks!)
+
 * `Q & A`

From noreply at buildbot.pypy.org  Fri Oct  4 11:24:44 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  4 Oct 2013 11:24:44 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: tweaks
Message-ID: <20131004092444.E269E1C014D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5068:0f46fb1f8767
Date: 2013-10-04 11:22 +0200
http://bitbucket.org/pypy/extradoc/changeset/0f46fb1f8767/

Log:	tweaks

diff --git a/talk/pyconza2013/talk.pdf b/talk/pyconza2013/talk.pdf
index 80a179ffd04df370eb9f786822977148c46dc6be..fec69aacfbd0fc9af5c9c60eb65501eed188fc5a
GIT binary patch

[cut]

diff --git a/talk/pyconza2013/talk.rst b/talk/pyconza2013/talk.rst
--- a/talk/pyconza2013/talk.rst
+++ b/talk/pyconza2013/talk.rst
@@ -14,6 +14,8 @@
 Introduction
 ------------
 
+* me: Armin Rigo
+
 * what is PyPy: an alternative implementation of Python
 
 * very compatible
@@ -25,7 +27,7 @@
 ------------
 
 .. image:: speed.png
-   :scale: 65%
+   :scale: 67%
    :align: center
 
 
@@ -138,9 +140,9 @@
 
 ::
 
-    with the_lock:      with the_lock:        with ..
-      x = obj.val         x = obj.val           x =..
-      obj.val = x + 1     obj.val = x + 1       obj..
+    with the_lock:     with the_lock:        with ..
+      x = obj.val        x = obj.val           x =..
+      obj.val = x + 1    obj.val = x + 1       obj..
 
 
 Locks != Transactions
@@ -148,9 +150,9 @@
 
 ::
 
-    with atomic:        with atomic:          with ..
-      x = obj.val         x = obj.val           x =..
-      obj.val = x + 1     obj.val = x + 1       obj..
+    with atomic:       with atomic:          with ..
+      x = obj.val        x = obj.val           x =..
+      obj.val = x + 1    obj.val = x + 1       obj..
 
 
 STM
@@ -192,9 +194,9 @@
 
 * no, it would be quite hard to implement it in standard CPython
 
-* but not completely impossible
+* too bad for now, only in PyPy
 
-* too bad for now, only in PyPy
+* but it would not be completely impossible
 
 
 But...
@@ -242,7 +244,7 @@
 
   for account in all_accounts:
      add_task(account.apply_interest)
-  run_tasks()
+  run_all_tasks()
 
 
 Internally

From noreply at buildbot.pypy.org  Fri Oct  4 21:28:48 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Fri,  4 Oct 2013 21:28:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix PyPy issue 1589
Message-ID: <20131004192848.A97851C05DF@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r67153:9e583c8e7b41
Date: 2013-10-04 21:24 +0200
http://bitbucket.org/pypy/pypy/changeset/9e583c8e7b41/

Log:	Fix PyPy issue 1589

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2965,6 +2965,12 @@
 
         assert len(list(a[0])) == 2
 
+    def test_issue_1589(self):
+        import numpypy as numpy
+        c = numpy.array([[(1, 2, 'a'), (3, 4, 'b')], [(5, 6, 'c'), (7, 8, 'd')]],
+                        dtype=[('bg', 'i8'), ('fg', 'i8'), ('char', 'S1')])
+        assert c[0][0]["char"] == 'a'
+
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1766,14 +1766,14 @@
 
     def store(self, arr, i, offset, box):
         assert isinstance(box, interp_boxes.W_StringBox)
-        # XXX simplify to range(box.dtype.get_size()) ?
         return self._store(arr.storage, i, offset, box)
 
     @jit.unroll_safe
     def _store(self, storage, i, offset, box):
         assert isinstance(box, interp_boxes.W_StringBox)
-        for k in range(min(self.size, box.arr.size-offset)):
-            storage[k + i] = box.arr.storage[k + offset]
+        # XXX simplify to range(box.dtype.get_size()) ?
+        for k in range(min(self.size, box.arr.size-box.ofs)):
+            storage[k + offset + i] = box.arr.storage[k + box.ofs]
 
     def read(self, arr, i, offset, dtype=None):
         if dtype is None:

From noreply at buildbot.pypy.org  Sat Oct  5 10:05:44 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  5 Oct 2013 10:05:44 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Actually even reads of GC pointers
 would need special care
Message-ID: <20131005080544.DA3531C01F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r67154:8206090afbcb
Date: 2013-10-05 07:55 +0200
http://bitbucket.org/pypy/pypy/changeset/8206090afbcb/

Log:	Actually even reads of GC pointers would need special care

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -798,7 +798,7 @@
 # behavior is fine, like incrementing some global counter.
 # XXX only for GC objects for now
 # XXX but it should replace 'stm_dont_track_raw_accesses' too
-# XXX DON'T USE for *writes* of a GC pointer into an object
+# XXX only for reads and writes of non-GC pointers
 stm_ignored = _StmIgnored()
 
 
diff --git a/rpython/translator/stm/writebarrier.py b/rpython/translator/stm/writebarrier.py
--- a/rpython/translator/stm/writebarrier.py
+++ b/rpython/translator/stm/writebarrier.py
@@ -134,6 +134,11 @@
                     raise Exception(
                         "%r: 'with stm_ignored:' contains unsupported "
                         "operation %r writing a GC pointer" % (graph, op))
+                if wants_a_barrier[op] == 'R' and is_getter and (
+                        is_gc_ptr(op.result.concretetype)):
+                    raise Exception(
+                        "%r: 'with stm_ignored:' contains unsupported "
+                        "operation %r reading a GC pointer" % (graph, op))
                 assert 'I' <= wants_a_barrier[op] < 'W'
                 wants_a_barrier[op] = 'I'
         #

From noreply at buildbot.pypy.org  Sat Oct  5 10:49:38 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Sat,  5 Oct 2013 10:49:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: interpreter/test2 doesn't exist any more
Message-ID: <20131005084938.C71701DAE70@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r67156:0026aa84a26b
Date: 2013-10-05 10:48 +0200
http://bitbucket.org/pypy/pypy/changeset/0026aa84a26b/

Log:	interpreter/test2 doesn't exist any more

diff --git a/pypy/pytest-A.py b/pypy/pytest-A.py
--- a/pypy/pytest-A.py
+++ b/pypy/pytest-A.py
@@ -5,7 +5,6 @@
     'arm': ['interpreter/astcompiler/test',
             'interpreter/pyparser/test',
             'interpreter/test',
-            'interpreter/test2',
             'module/test_lib_pypy',
             'objspace/std/test',
     ],

From noreply at buildbot.pypy.org  Sat Oct  5 10:49:37 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Sat,  5 Oct 2013 10:49:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: Avoid a spurious failure when TMPDIR is
 inside the source
Message-ID: <20131005084937.A6B941C01B0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r67155:733feca87f89
Date: 2013-10-05 10:48 +0200
http://bitbucket.org/pypy/pypy/changeset/733feca87f89/

Log:	Avoid a spurious failure when TMPDIR is inside the source

diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -954,6 +954,8 @@
 
         sys.path.append(self.goal_dir)
         # make sure cwd does not contain a stdlib
+        if self.tmp_dir.startswith(self.trunkdir):
+            skip('TMPDIR is inside the PyPy source')
         os.chdir(self.tmp_dir)
         tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c')
         try:

From noreply at buildbot.pypy.org  Sat Oct  5 12:31:58 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  5 Oct 2013 12:31:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: (austiine, arigo)
Message-ID: <20131005103158.43C2C1C02D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67157:d7d63baf7ea4
Date: 2013-10-05 12:31 +0200
http://bitbucket.org/pypy/pypy/changeset/d7d63baf7ea4/

Log:	(austiine, arigo)

	issue1618: this should fix the performance degradation of 'pow(huge,
	smallish, smallish)'. Note that CPython could use the same too.

diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -734,7 +734,9 @@
             # if base < 0:
             #     base = base % modulus
             # Having the base positive just makes things easier.
-            if a.sign < 0:
+            # As a (very good) optimization, we also reduce 'base' here
+            # if it is much bigger than the modulus.
+            if a.sign < 0 or a.numdigits() > c.numdigits():
                 a = a.mod(c)
 
         elif b.sign == 0:

From noreply at buildbot.pypy.org  Sat Oct  5 12:34:39 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  5 Oct 2013 12:34:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: Yay, we did ARM too!  ...long ago
Message-ID: <20131005103439.762D71C02D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67158:63b0e347b780
Date: 2013-10-05 12:33 +0200
http://bitbucket.org/pypy/pypy/changeset/63b0e347b780/

Log:	Yay, we did ARM too! ...long ago

diff --git a/pypy/TODO b/pypy/TODO
deleted file mode 100644
--- a/pypy/TODO
+++ /dev/null
@@ -1,2 +0,0 @@
-
-* ARM

From noreply at buildbot.pypy.org  Sat Oct  5 13:25:12 2013
From: noreply at buildbot.pypy.org (jerith)
Date: Sat,  5 Oct 2013 13:25:12 +0200 (CEST)
Subject: [pypy-commit] cffi default: (arigo, fijal,
 jerith) More complicated (and better) juggling of compiler args for
 gcc/clang in verify tests.
Message-ID: <20131005112513.003E01C01F4@cobra.cs.uni-duesseldorf.de>

Author: Jeremy Thurgood <firxen at gmail.com>
Branch: 
Changeset: r1350:a88e541709e8
Date: 2013-10-05 13:21 +0200
http://bitbucket.org/cffi/cffi/changeset/a88e541709e8/

Log:	(arigo, fijal, jerith) More complicated (and better) juggling of
	compiler args for gcc/clang in verify tests.

diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -6,17 +6,20 @@
 
 if sys.platform == 'win32':
     pass      # no obvious -Werror equivalent on MSVC
-elif (sys.platform == 'darwin' and
-      [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
-    pass      # recent MacOSX come with clang by default, and passing some
-              # flags from the interpreter (-mno-fused-madd) generates a
-              # warning --- which is interpreted as an error with -Werror
 else:
-    # assume a standard GCC
+    if (sys.platform == 'darwin' and
+          [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
+        # special things for clang
+        extra_compile_args = [
+            '-Werror', '-Qunused-arguments', '-Wno-error=shorten-64-to-32']
+    else:
+        # assume a standard gcc
+        extra_compile_args = ['-Werror']
+
     class FFI(FFI):
         def verify(self, *args, **kwds):
             return super(FFI, self).verify(
-                *args, extra_compile_args=['-Werror'], **kwds)
+                *args, extra_compile_args=extra_compile_args, **kwds)
 
 def setup_module():
     import cffi.verifier

From noreply at buildbot.pypy.org  Sat Oct  5 17:10:14 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  5 Oct 2013 17:10:14 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Fix: cannot put the BUSY flag in the
 same counter modified in
Message-ID: <20131005151014.C9C0B1C02D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r67159:e80585133345
Date: 2013-10-05 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/e80585133345/

Log:	Fix: cannot put the BUSY flag in the same counter modified in
	'stm_ignored' mode. We end up in cases where the guard's code is
	compiled twice, which crashes later in the x86 backend.

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -498,6 +498,9 @@
     rd_virtuals = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
 
+    rd_stm_busy = False       # same as CNT_BUSY_FLAG, in a different field,
+                              # only for stm
+
     CNT_BASE_MASK  =  0x0FFFFFFF     # the base counter value
     CNT_BUSY_FLAG  =  0x10000000     # if set, busy tracing from the guard
     CNT_TYPE_MASK  =  0x60000000     # mask for the type
@@ -559,35 +562,35 @@
     _trace_and_compile_from_bridge._dont_inline_ = True
 
     def must_compile(self, deadframe, metainterp_sd, jitdriver_sd):
-        ok = self.must_compile_approx(deadframe, metainterp_sd, jitdriver_sd)
-        if not rgc.stm_is_enabled():
-            return ok
+        if rgc.stm_is_enabled():
+            method = self.must_compile_stm
         else:
-            # in stm mode, the return value may (rarely) be True even if a
-            # real, stm-protected read of self._counter says "busy".
-            return ok and not (self._counter & self.CNT_BUSY_FLAG)
+            method = self.must_compile_nonstm
+        return method(deadframe, metainterp_sd, jitdriver_sd)
 
-    def must_compile_approx(self, deadframe, metainterp_sd, jitdriver_sd):
+    def must_compile_stm(self, deadframe, metainterp_sd, jitdriver_sd):
+        trace_eagerness = jitdriver_sd.warmstate.trace_eagerness
+        with stm_ignored:
+            approx_counter = self._counter + 1
+            self._counter = approx_counter
+        return approx_counter >= trace_eagerness and not self.rd_stm_busy
+
+    def must_compile_nonstm(self, deadframe, metainterp_sd, jitdriver_sd):
         trace_eagerness = jitdriver_sd.warmstate.trace_eagerness
         #
-        with stm_ignored:
-            approx_counter = self._counter
-        if approx_counter <= self.CNT_BASE_MASK:
+        if self._counter <= self.CNT_BASE_MASK:
             # simple case: just counting from 0 to trace_eagerness
-            approx_counter += 1
-            with stm_ignored:
-                self._counter = approx_counter
-            return approx_counter >= trace_eagerness
+            self._counter += 1
+            return self._counter >= trace_eagerness
         #
         # do we have the BUSY flag?  If so, we're tracing right now, e.g. in an
         # outer invocation of the same function, so don't trace again for now.
-        elif approx_counter & self.CNT_BUSY_FLAG:
+        elif self._counter & self.CNT_BUSY_FLAG:
             return False
         #
         else: # we have a GUARD_VALUE that fails.  Make a _counters instance
             # (only now, when the guard is actually failing at least once),
             # and use it to record some statistics about the failing values.
-            assert not rgc.stm_is_enabled(), "XXX"
             index = self._counter & self.CNT_BASE_MASK
             typetag = self._counter & self.CNT_TYPE_MASK
             counters = self._counters
@@ -621,7 +624,10 @@
 
     def start_compiling(self):
         # start tracing and compiling from this guard.
-        self._counter |= self.CNT_BUSY_FLAG
+        if rgc.stm_is_enabled():
+            self.rd_stm_busy = True
+        else:
+            self._counter |= self.CNT_BUSY_FLAG
 
     def done_compiling(self):
         # done tracing and compiling from this guard.  Either the bridge has
@@ -629,8 +635,12 @@
         # in self._counter will not be seen any more, or not, in which case
         # we should reset the counter to 0, in order to wait a bit until the
         # next attempt.
-        if self._counter >= 0:
+        if rgc.stm_is_enabled():
+            self.rd_stm_busy = False
             self._counter = 0
+        else:
+            if self._counter >= 0:
+                self._counter = 0
         self._counters = None
 
     def compile_and_attach(self, metainterp, new_loop):

From noreply at buildbot.pypy.org  Sat Oct  5 23:15:40 2013
From: noreply at buildbot.pypy.org (jerith)
Date: Sat,  5 Oct 2013 23:15:40 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: FFIBuilder implementation.
Message-ID: <20131005211540.691E21C01B0@cobra.cs.uni-duesseldorf.de>

Author: Jeremy Thurgood <firxen at gmail.com>
Branch: split-verify
Changeset: r1351:eb2e8acaa3db
Date: 2013-10-05 23:11 +0200
http://bitbucket.org/cffi/cffi/changeset/eb2e8acaa3db/

Log:	FFIBuilder implementation.

diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -1,7 +1,7 @@
 __all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError',
            'FFIError']
 
-from .api import FFI, CDefError, FFIError
+from .api import FFI, CDefError, FFIError, FFIBuilder
 from .ffiplatform import VerificationError, VerificationMissing
 
 __version__ = "0.7.2"
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -448,3 +448,68 @@
         return None
     else:
         return ffi._get_cached_btype(tp)
+
+
+class FFIBuilder(object):
+    def __init__(self, module_name, module_path, backend=None):
+        self._module_name = module_name
+        self._module_path = module_path
+        self.ffi = FFI(backend=backend)
+        self._module_source = "\n".join([
+            "from cffi import FFI",
+            "",
+            "ffi = FFI()",
+            "",
+        ])
+
+    def cdef(self, csource, override=False):
+        self.ffi.cdef(csource, override=override)
+        self._module_source += "ffi.cdef(%r, override=%r)\n" % (
+            csource, override)
+
+    def add_dlopen(self, libname, name, flags=0):
+        lib = self.ffi.dlopen(name, flags=flags)
+        self._module_source += '\n'.join([
+            "def load_%s():",
+            "    return ffi.dlopen(%r, flags=%r)",
+            "",
+        ]) % (libname, name, flags)
+        return lib
+
+    def makelib(self, libname, source='', **kwargs):
+        # XXX: We use force_generic_engine here because vengine_cpy collects
+        #      types when it writes the source.
+        import os.path
+        from .verifier import Verifier, _caller_dir_pycache, _get_so_suffix
+        tmpdir = _caller_dir_pycache()
+        self.ffi.verifier = Verifier(
+            self.ffi, source, tmpdir, libname, force_generic_engine=True,
+            **kwargs)
+        libfilename = libname + _get_so_suffix()
+        self.ffi.verifier.make_library(
+            os.path.join(self._module_path, libfilename))
+        self._module_source += '\n'.join([
+            "def load_%s():",
+            "    from cffi.verifier import Verifier",
+            "    import os.path",
+            "    module_path = os.path.dirname(__file__)",
+            "    verifier = Verifier(",
+            "        ffi, None, module_path, %r, force_generic_engine=True)",
+            "    verifier._has_module = True",
+            "    return verifier._load_library()",
+            "",
+        ]) % (libname, libname)
+
+    def write_ffi_module(self):
+        import os
+        try:
+            os.makedirs(self._module_path)
+        except OSError:
+            pass
+
+        module_filename = self._module_name + '.py'
+        file = open(os.path.join(self._module_path, module_filename), 'w')
+        try:
+            file.write(self._module_source)
+        finally:
+            file.close()
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -55,6 +55,17 @@
             self._write_source()
         self._compile_module()
 
+    def make_library(self, libraryfilename):
+        if not self._has_module:
+            self.compile_module()
+        try:
+            same = ffiplatform.samefile(self.modulefilename, libraryfilename)
+        except OSError:
+            same = False
+        if not same:
+            _ensure_dir(libraryfilename)
+            shutil.copy(self.modulefilename, libraryfilename)
+
     def load_library(self):
         """Get a C module from this Verifier instance.
         Returns an instance of a FFILibrary class that behaves like the
diff --git a/testing/test_makelib.py b/testing/test_makelib.py
new file mode 100644
--- /dev/null
+++ b/testing/test_makelib.py
@@ -0,0 +1,69 @@
+import math
+import sys
+from cffi import FFIBuilder
+
+
+def test_ffibuilder_makelib(tmpdir):
+    builder = FFIBuilder("foo_ffi", str(tmpdir))
+    builder.cdef("""
+        double sin(double x);
+    """)
+    builder.makelib('foo', '#include <math.h>')
+    builder.write_ffi_module()
+
+    sys.path.append(str(tmpdir))
+    try:
+        import foo_ffi
+    finally:
+        sys.path.remove(str(tmpdir))
+        for name in sys.modules.keys():
+            if name.endswith('foo_ffi'):
+                sys.modules.pop(name)
+
+    lib = foo_ffi.load_foo()
+    assert lib.sin(12.3) == math.sin(12.3)
+
+
+def test_ffibuilder_dlopen(tmpdir):
+    builder = FFIBuilder("foo_ffi", str(tmpdir))
+    builder.cdef("""
+        double sin(double x);
+    """)
+    builder.add_dlopen('foo', "m")
+    builder.write_ffi_module()
+
+    sys.path.append(str(tmpdir))
+    try:
+        import foo_ffi
+    finally:
+        sys.path.remove(str(tmpdir))
+        for name in sys.modules.keys():
+            if name.endswith('foo_ffi'):
+                sys.modules.pop(name)
+
+    lib = foo_ffi.load_foo()
+    assert lib.sin(12.3) == math.sin(12.3)
+
+
+def test_ffibuilder_makelib_and_dlopen(tmpdir):
+    builder = FFIBuilder("foo_ffi", str(tmpdir))
+    builder.cdef("""
+        double sin(double x);
+    """)
+    builder.makelib('foo', '#include <math.h>')
+    builder.add_dlopen('bar', "m")
+    builder.write_ffi_module()
+
+    sys.path.append(str(tmpdir))
+    try:
+        import foo_ffi
+    finally:
+        sys.path.remove(str(tmpdir))
+        for name in sys.modules.keys():
+            if name.endswith('foo_ffi'):
+                sys.modules.pop(name)
+
+    lib_foo = foo_ffi.load_foo()
+    assert lib_foo.sin(12.3) == math.sin(12.3)
+    lib_bar = foo_ffi.load_bar()
+    assert lib_bar.sin(12.3) == math.sin(12.3)

From noreply at buildbot.pypy.org  Sun Oct  6 08:44:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  6 Oct 2013 08:44:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update the comment from CPython's
	101bf827611a.
Message-ID: <20131006064451.E5E921C0223@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67160:0bb221958198
Date: 2013-10-06 08:44 +0200
http://bitbucket.org/pypy/pypy/changeset/0bb221958198/

Log:	Update the comment from CPython's 101bf827611a.

diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -731,11 +731,15 @@
             if c.numdigits() == 1 and c._digits[0] == ONEDIGIT:
                 return NULLRBIGINT
 
-            # if base < 0:
-            #     base = base % modulus
-            # Having the base positive just makes things easier.
-            # As a (very good) optimization, we also reduce 'base' here
-            # if it is much bigger than the modulus.
+            # Reduce base by modulus in some cases:
+            # 1. If base < 0.  Forcing the base non-neg makes things easier.
+            # 2. If base is obviously larger than the modulus.  The "small
+            #    exponent" case later can multiply directly by base repeatedly,
+            #    while the "large exponent" case multiplies directly by base 31
+            #    times.  It can be unboundedly faster to multiply by
+            #    base % modulus instead.
+            # We could _always_ do this reduction, but mod() isn't cheap,
+            # so we only do it when it buys something.
             if a.sign < 0 or a.numdigits() > c.numdigits():
                 a = a.mod(c)
 

From noreply at buildbot.pypy.org  Sun Oct  6 09:50:15 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Sun,  6 Oct 2013 09:50:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: ARMv4 doesn't have BLX. Add support for
	older ARM
Message-ID: <20131006075015.523681C01B0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r67161:f1e9c4f357e1
Date: 2013-10-06 09:49 +0200
http://bitbucket.org/pypy/pypy/changeset/f1e9c4f357e1/

Log:	ARMv4 doesn't have BLX. Add support for older ARM

diff --git a/rpython/translator/c/src/stacklet/switch_arm_gcc.h b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
--- a/rpython/translator/c/src/stacklet/switch_arm_gcc.h
+++ b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
@@ -1,3 +1,10 @@
+#if __ARM_ARCH__ >= 5
+# define call_reg(x) "blx " #x "\n"
+#elif defined (__ARM_ARCH_4T__)
+# define call_reg(x) "mov lr, pc ; bx " #x "\n"
+#else
+# define call_reg(x) "mov lr, pc ; mov pc, " #x "\n"
+#endif
 
 static void __attribute__((optimize("O3"))) *slp_switch(void *(*save_state)(void*, void*),
                         void *(*restore_state)(void*, void*),
@@ -11,7 +18,7 @@
     "mov r5, %[extra]\n"
     "mov r0, sp\n"        	/* arg 1: current (old) stack pointer */
     "mov r1, r5\n"        	/* arg 2: extra                       */
-    "blx r3\n"				/* call save_state()                  */
+    call_reg(r3)		/* call save_state()                  */
 
     /* skip the rest if the return value is null */
     "cmp r0, #0\n"
@@ -23,7 +30,7 @@
 	stack is not restored yet.  It contains only garbage here. */
     "mov r1, r5\n"       	/* arg 2: extra                       */
    	 						/* arg 1: current (new) stack pointer is already in r0*/
-    "blx r4\n"           	/* call restore_state()               */
+    call_reg(r4)		/* call restore_state()               */
 
     /* The stack's content is now restored. */
     "zero:\n"

From noreply at buildbot.pypy.org  Sun Oct  6 10:46:43 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  6 Oct 2013 10:46:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add an assert that seems to fail in
	issue1619
Message-ID: <20131006084643.3D1E21C1152@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67162:7e05f0998a32
Date: 2013-10-06 10:45 +0200
http://bitbucket.org/pypy/pypy/changeset/7e05f0998a32/

Log:	Add an assert that seems to fail in issue1619

diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -783,6 +783,7 @@
         v = self.virtuals_cache.get_int(index)
         if not v:
             v = self.rd_virtuals[index]
+            ll_assert(bool(v), "resume.py: null rd_virtuals[index]")
             assert v.is_about_raw and isinstance(v, VRawBufferStateInfo)
             v = v.allocate_int(self, index)
             ll_assert(v == self.virtuals_cache.get_int(index), "resume.py: bad cache")

From noreply at buildbot.pypy.org  Sun Oct  6 10:51:55 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  6 Oct 2013 10:51:55 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: (fijal, arigo) work on fileops
Message-ID: <20131006085155.AAC291C1152@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67163:7c6969e0bd84
Date: 2013-10-06 10:51 +0200
http://bitbucket.org/pypy/pypy/changeset/7c6969e0bd84/

Log:	(fijal, arigo) work on fileops

diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py
--- a/rpython/flowspace/specialcase.py
+++ b/rpython/flowspace/specialcase.py
@@ -1,3 +1,4 @@
+import os
 from rpython.flowspace.model import Constant, const
 
 SPECIAL_CASES = {}
@@ -37,6 +38,18 @@
     return space.frame.do_operation('simple_call', const(isinstance),
             w_instance, w_type)
 
+ at register_flow_sc(open)
+def sc_open(space, *args_w):
+    from rpython.rlib.rfile import create_file
+
+    return space.frame.do_operation("simple_call", const(create_file), *args_w)
+
+ at register_flow_sc(os.tmpfile)
+def sc_os_tmpfile(space):
+    from rpython.rlib.rfile import create_temp_rfile
+
+    return space.frame.do_operation("simple_call", const(create_temp_rfile))
+
 # _________________________________________________________________________
 # a simplified version of the basic printing routines, for RPython programs
 class StdOutBuffer:
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -1,55 +1,158 @@
 
-""" This file makes open() and friends RPython
+""" This file makes open() and friends RPython. Note that RFile should not
+be used directly and instead it's magically appearing each time you call
+python builtin open()
 """
 
 import os
-from rpython.annotator.model import SomeObject, SomeString, SomeInteger
-from rpython.rtyper.extregistry import ExtRegistryEntry
-from rpython.rtyper.extfunc import register_external
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib import rposix
+from rpython.rlib.rstring import StringBuilder
 
-class SomeFile(SomeObject):
-    def method_write(self, s_arg):
-        assert isinstance(s_arg, SomeString)
+eci = ExternalCompilationInfo(includes=['stdio.h'])
 
-    def method_read(self, s_arg=None):
-        if s_arg is not None:
-            assert isinstance(s_arg, SomeInteger)
-        return SomeString(can_be_None=False)
+def llexternal(*args):
+    return rffi.llexternal(*args, compilation_info=eci)
 
-    def method_close(self):
-        pass
+FILE = lltype.Struct('FILE') # opaque type maybe
 
-    def method_seek(self, s_arg, s_whence=None):
-        assert isinstance(s_arg, SomeInteger)
-        if s_whence is not None:
-            assert isinstance(s_whence, SomeInteger)
+c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
+c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
+c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
+                                     lltype.Ptr(FILE)], rffi.SIZE_T)
+c_read = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
+                                   lltype.Ptr(FILE)], rffi.SIZE_T)
+c_feof = llexternal('feof', [lltype.Ptr(FILE)], rffi.INT)
+c_ferror = llexternal('ferror', [lltype.Ptr(FILE)], rffi.INT)
+c_clearerror = llexternal('clearerr', [lltype.Ptr(FILE)], lltype.Void)
+c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT],
+                          rffi.INT)
+c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
+c_fileno = llexternal('fileno', [lltype.Ptr(FILE)], rffi.INT)
+c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], lltype.Signed)
+c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], lltype.Signed)
 
-    def rtyper_makekey(self):
-        return self.__class__,
+BASE_BUF_SIZE = 4096
 
-    def rtyper_makerepr(self, rtyper):
-        from rpython.rtyper.lltypesystem.rfile import FileRepr
+def create_file(filename, mode="r", buffering=-1):
+    assert buffering == -1
+    assert filename is not None
+    assert mode is not None
+    ll_name = rffi.str2charp(filename)
+    try:
+        ll_mode = rffi.str2charp(mode)
+        try:
+            ll_f = c_open(ll_name, ll_mode)
+            if not ll_f:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+        finally:
+            lltype.free(ll_mode, flavor='raw')
+    finally:
+        lltype.free(ll_name, flavor='raw')
+    return RFile(ll_f)
 
-        return FileRepr(rtyper)
+def create_temp_rfile():
+    res = c_tmpfile()
+    if not res:
+        errno = rposix.get_errno()
+        raise OSError(errno, os.strerror(errno))
+    return RFile(res)
 
-class FileEntry(ExtRegistryEntry):
-    _about_ = open
+class RFile(object):
+    def __init__(self, ll_file):
+        self.ll_file = ll_file
 
-    def compute_result_annotation(self, s_name, s_mode=None):
-        assert isinstance(s_name, SomeString)
-        if s_mode is not None:
-            assert isinstance(s_mode, SomeString)
-        return SomeFile()
+    def write(self, value):
+        assert value is not None
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        assert value is not None
+        ll_value = rffi.get_nonmovingbuffer(value)
+        try:
+            # note that since we got a nonmoving buffer, it is either raw
+            # or already cannot move, so the arithmetics below are fine
+            total_bytes = 0
+            ll_current = ll_value
+            while total_bytes < len(value):
+                bytes = c_write(ll_current, 1, len(value) - r_uint(total_bytes),
+                                ll_file)
+                if bytes == 0:
+                    errno = rposix.get_errno()
+                    raise OSError(errno, os.strerror(errno))
+                total_bytes += bytes
+                ll_current = rffi.cast(rffi.CCHARP,
+                                       rffi.cast(lltype.Unsigned, ll_value) +
+                                       total_bytes)
+        finally:
+            rffi.free_nonmovingbuffer(value, ll_value)
 
-    def specialize_call(self, hop):
-        return hop.r_result.rtype_constructor(hop)
+    def close(self):
+        if self.ll_file:
+            # double close is allowed
+            res = c_close(self.ll_file)
+            self.ll_file = lltype.nullptr(FILE)
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
 
-class OSTempfileEntry(ExtRegistryEntry):
-    _about_ = os.tmpfile
+    def read(self, size=-1):
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        if size < 0:
+            # read the entire contents
+            buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
+            try:
+                s = StringBuilder()
+                while True:
+                    returned_size = c_read(buf, 1, BASE_BUF_SIZE, ll_file)
+                    if returned_size == 0:
+                        if c_feof(ll_file):
+                            # ok, finished
+                            return s.build()
+                        errno = c_ferror(ll_file)
+                        c_clearerror(ll_file)
+                        raise OSError(errno, os.strerror(errno))
+                    s.append_charpsize(buf, returned_size)
+            finally:
+                lltype.free(buf, flavor='raw')
+        else:
+            raw_buf, gc_buf = rffi.alloc_buffer(size)
+            try:
+                returned_size = c_read(raw_buf, 1, size, ll_file)
+                if returned_size == 0:
+                    if not c_feof(ll_file):
+                        errno = c_ferror(ll_file)
+                        raise OSError(errno, os.strerror(errno))
+                s = rffi.str_from_buffer(raw_buf, gc_buf, size,
+                                         rffi.cast(lltype.Signed, returned_size))
+            finally:
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+            return s
 
-    def compute_result_annotation(self):
-        return SomeFile()
+    def seek(self, pos, whence=0):
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        res = c_fseek(ll_file, pos, whence)
+        if res == -1:
+            errno = rposix.get_errno()
+            raise OSError(errno, os.strerror(errno))
 
-    def specialize_call(self, hop):
-        return hop.r_result.rtype_tempfile(hop)
+    def fileno(self):
+        if self.ll_file:
+            return intmask(c_fileno(self.ll_file))
+        raise ValueError("I/O operation on closed file")
 
+    def tell(self):
+        if self.ll_file:
+            res = intmask(c_ftell(self.ll_file))
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return res
+        raise ValueError("I/O operation on closed file")
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -78,3 +78,31 @@
 
         f()
         self.interpret(f, [])
+
+    def test_fileno(self):
+        fname = str(self.tmpdir.join('file_5'))
+
+        def f():
+            f = open(fname, "w")
+            try:
+                return f.fileno()
+            finally:
+                f.close()
+
+        res = self.interpret(f, [])
+        assert res > 2
+
+    def test_tell(self):
+        fname = str(self.tmpdir.join('file_tell'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            try:
+                return f.tell()
+            finally:
+                f.close()
+
+        res = self.interpret(f, [])
+        assert res == 3
+
diff --git a/rpython/rtyper/lltypesystem/rfile.py b/rpython/rtyper/lltypesystem/rfile.py
deleted file mode 100644
--- a/rpython/rtyper/lltypesystem/rfile.py
+++ /dev/null
@@ -1,195 +0,0 @@
-
-import os
-from rpython.rlib import rposix
-from rpython.rlib.rarithmetic import r_uint
-from rpython.annotator import model as annmodel
-from rpython.rtyper.rtyper import Repr
-from rpython.rlib.rstring import StringBuilder
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
-from rpython.rtyper.lltypesystem.rstr import string_repr, STR
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.annlowlevel import hlstr
-from rpython.rtyper.lltypesystem.lloperation import llop
-
-FILE = lltype.Struct('FILE') # opaque type maybe
-FILE_WRAPPER = lltype.GcStruct("FileWrapper", ('file', lltype.Ptr(FILE)))
-
-eci = ExternalCompilationInfo(includes=['stdio.h'])
-
-def llexternal(*args):
-    return rffi.llexternal(*args, compilation_info=eci)
-
-c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
-c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
-c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
-                                     lltype.Ptr(FILE)], rffi.SIZE_T)
-c_read = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
-                                   lltype.Ptr(FILE)], rffi.SIZE_T)
-c_feof = llexternal('feof', [lltype.Ptr(FILE)], rffi.INT)
-c_ferror = llexternal('ferror', [lltype.Ptr(FILE)], rffi.INT)
-c_clearerror = llexternal('clearerr', [lltype.Ptr(FILE)], lltype.Void)
-c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT],
-                          rffi.INT)
-c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
-
-def ll_open(name, mode):
-    file_wrapper = lltype.malloc(FILE_WRAPPER)
-    ll_name = rffi.str2charp(name)
-    ll_mode = rffi.str2charp(mode)
-    try:
-        ll_f = c_open(ll_name, ll_mode)
-        if not ll_f:
-            errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
-        file_wrapper.file = ll_f
-    finally:
-        lltype.free(ll_name, flavor='raw')
-        lltype.free(ll_mode, flavor='raw')
-    return file_wrapper
-
-def ll_tmpfile():
-    file_wrapper = lltype.malloc(FILE_WRAPPER)
-    res = c_tmpfile()
-    if not res:
-        errno = rposix.get_errno()
-        raise OSError(errno, os.strerror(errno))
-    file_wrapper.file = res
-    return file_wrapper
-
-def ll_write(file_wrapper, value):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    value = hlstr(value)
-    assert value is not None
-    ll_value = rffi.get_nonmovingbuffer(value)
-    try:
-        # note that since we got a nonmoving buffer, it is either raw
-        # or already cannot move, so the arithmetics below are fine
-        total_bytes = 0
-        ll_current = ll_value
-        while total_bytes < len(value):
-            bytes = c_write(ll_current, 1, len(value) - r_uint(total_bytes),
-                            ll_file)
-            if bytes == 0:
-                errno = rposix.get_errno()
-                raise OSError(errno, os.strerror(errno))
-            total_bytes += bytes
-            ll_current = rffi.cast(rffi.CCHARP,
-                                   rffi.cast(lltype.Unsigned, ll_value) +
-                                   total_bytes)
-    finally:
-        rffi.free_nonmovingbuffer(value, ll_value)
-
-BASE_BUF_SIZE = 4096
-
-def ll_read(file_wrapper, size):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    if size < 0:
-        # read the entire contents
-        buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
-        try:
-            s = StringBuilder()
-            while True:
-                returned_size = c_read(buf, 1, BASE_BUF_SIZE, ll_file)
-                if returned_size == 0:
-                    if c_feof(ll_file):
-                        # ok, finished
-                        return s.build()
-                    errno = c_ferror(ll_file)
-                    c_clearerror(ll_file)
-                    raise OSError(errno, os.strerror(errno))
-                s.append_charpsize(buf, returned_size)
-        finally:
-            lltype.free(buf, flavor='raw')
-    else:
-        raw_buf, gc_buf = rffi.alloc_buffer(size)
-        try:
-            returned_size = c_read(raw_buf, 1, size, ll_file)
-            if returned_size == 0:
-                if not c_feof(ll_file):
-                    errno = c_ferror(ll_file)
-                    raise OSError(errno, os.strerror(errno))
-            s = rffi.str_from_buffer(raw_buf, gc_buf, size,
-                                     rffi.cast(lltype.Signed, returned_size))
-        finally:
-            rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
-        return s
-def ll_seek(file_wrapper, pos, whence):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    res = c_fseek(ll_file, pos, whence)
-    if res == -1:
-        errno = rposix.get_errno()
-        raise OSError(errno, os.strerror(errno))
-
-def ll_close(file_wrapper):
-    if file_wrapper.file:
-        # double close is allowed
-        res = c_close(file_wrapper.file)
-        file_wrapper.file = lltype.nullptr(FILE)
-        if res == -1:
-            errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
-
-class FileRepr(Repr):
-    lowleveltype = lltype.Ptr(FILE_WRAPPER)
-
-    def __init__(self, typer):
-        Repr.__init__(self)
-
-    def rtype_constructor(self, hop):
-        repr = hop.rtyper.getrepr(annmodel.SomeString())
-        arg_0 = hop.inputarg(repr, 0)
-        if len(hop.args_v) == 1:
-            arg_1 = hop.inputconst(string_repr, "r")
-        else:
-            arg_1 = hop.inputarg(repr, 1)
-        hop.exception_is_here()
-        open = hop.rtyper.getannmixlevel().delayedfunction(
-            ll_open, [annmodel.SomeString()] * 2,
-            annmodel.SomePtr(self.lowleveltype))
-        v_open = hop.inputconst(lltype.typeOf(open), open)
-        return hop.genop('direct_call', [v_open, arg_0, arg_1],
-                         resulttype=self)
-
-    def rtype_tempfile(self, hop):
-        tmpfile = hop.rtyper.getannmixlevel().delayedfunction(
-            ll_tmpfile, [], annmodel.SomePtr(self.lowleveltype))
-        v_tmpfile = hop.inputconst(lltype.typeOf(tmpfile), tmpfile)
-        hop.exception_is_here()
-        return hop.genop('direct_call', [v_tmpfile], resulttype=self)
-
-
-    def rtype_method_write(self, hop):
-        args_v = hop.inputargs(self, string_repr)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_write, *args_v)
-
-    def rtype_method_close(self, hop):
-        r_self = hop.inputarg(self, 0)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_close, r_self)
-
-    def rtype_method_read(self, hop):
-        r_self = hop.inputarg(self, 0)
-        if len(hop.args_v) != 2:
-            arg_1 = hop.inputconst(lltype.Signed, -1)
-        else:
-            arg_1 = hop.inputarg(lltype.Signed, 1)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_read, r_self, arg_1)
-
-    def rtype_method_seek(self, hop):
-        r_self = hop.inputarg(self, 0)
-        arg_1 = hop.inputarg(lltype.Signed, 1)
-        if len(hop.args_v) != 3:
-            arg_2 = hop.inputconst(lltype.Signed, os.SEEK_SET)
-        else:
-            arg_2 = hop.inputarg(lltype.Signed, 2)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_seek, r_self, arg_1, arg_2)
-

From noreply at buildbot.pypy.org  Sun Oct  6 12:19:45 2013
From: noreply at buildbot.pypy.org (jerith)
Date: Sun,  6 Oct 2013 12:19:45 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Clean up a few things,
	fix python3 compat.
Message-ID: <20131006101945.33D6F1C0223@cobra.cs.uni-duesseldorf.de>

Author: Jeremy Thurgood <firxen at gmail.com>
Branch: split-verify
Changeset: r1352:34923698f6a2
Date: 2013-10-06 12:19 +0200
http://bitbucket.org/cffi/cffi/changeset/34923698f6a2/

Log:	Clean up a few things, fix python3 compat.

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -480,14 +480,12 @@
         # XXX: We use force_generic_engine here because vengine_cpy collects
         #      types when it writes the source.
         import os.path
-        from .verifier import Verifier, _caller_dir_pycache, _get_so_suffix
-        tmpdir = _caller_dir_pycache()
+        from .verifier import Verifier, _get_so_suffix
         self.ffi.verifier = Verifier(
-            self.ffi, source, tmpdir, libname, force_generic_engine=True,
-            **kwargs)
-        libfilename = libname + _get_so_suffix()
+            self.ffi, source, force_generic_engine=True, **kwargs)
+        libfilename = '_'.join([self._module_name, libname])
         self.ffi.verifier.make_library(
-            os.path.join(self._module_path, libfilename))
+            os.path.join(self._module_path, libfilename + _get_so_suffix()))
         self._module_source += '\n'.join([
             "def load_%s():",
             "    from cffi.verifier import Verifier",
@@ -498,7 +496,7 @@
             "    verifier._has_module = True",
             "    return verifier._load_library()",
             "",
-        ]) % (libname, libname)
+        ]) % (libname, libfilename)
 
     def write_ffi_module(self):
         import os
diff --git a/testing/test_makelib.py b/testing/test_makelib.py
--- a/testing/test_makelib.py
+++ b/testing/test_makelib.py
@@ -3,6 +3,13 @@
 from cffi import FFIBuilder
 
 
+def _clean_modules(tmpdir, module_name):
+    sys.path.remove(str(tmpdir))
+    for name in list(sys.modules.keys()):
+        if name and name.endswith(module_name):
+            sys.modules.pop(name)
+
+
 def test_ffibuilder_makelib(tmpdir):
     builder = FFIBuilder("foo_ffi", str(tmpdir))
     builder.cdef("""
@@ -15,10 +22,7 @@
     try:
         import foo_ffi
     finally:
-        sys.path.remove(str(tmpdir))
-        for name in sys.modules.keys():
-            if name.endswith('foo_ffi'):
-                sys.modules.pop(name)
+        _clean_modules(tmpdir, 'foo_ffi')
 
     lib = foo_ffi.load_foo()
     assert lib.sin(12.3) == math.sin(12.3)
@@ -36,10 +40,7 @@
     try:
         import foo_ffi
     finally:
-        sys.path.remove(str(tmpdir))
-        for name in sys.modules.keys():
-            if name.endswith('foo_ffi'):
-                sys.modules.pop(name)
+        _clean_modules(tmpdir, 'foo_ffi')
 
     lib = foo_ffi.load_foo()
     assert lib.sin(12.3) == math.sin(12.3)
@@ -58,10 +59,7 @@
     try:
         import foo_ffi
     finally:
-        sys.path.remove(str(tmpdir))
-        for name in sys.modules.keys():
-            if name.endswith('foo_ffi'):
-                sys.modules.pop(name)
+        _clean_modules(tmpdir, 'foo_ffi')
 
     lib_foo = foo_ffi.load_foo()
     assert lib_foo.sin(12.3) == math.sin(12.3)

From noreply at buildbot.pypy.org  Sun Oct  6 13:13:52 2013
From: noreply at buildbot.pypy.org (jerith)
Date: Sun,  6 Oct 2013 13:13:52 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Collect built filepaths.
Message-ID: <20131006111352.D4B981C1190@cobra.cs.uni-duesseldorf.de>

Author: Jeremy Thurgood <firxen at gmail.com>
Branch: split-verify
Changeset: r1353:e179cc8d6f7a
Date: 2013-10-06 13:13 +0200
http://bitbucket.org/cffi/cffi/changeset/e179cc8d6f7a/

Log:	Collect built filepaths.

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -455,6 +455,7 @@
         self._module_name = module_name
         self._module_path = module_path
         self.ffi = FFI(backend=backend)
+        self._built_files = []
         self._module_source = "\n".join([
             "from cffi import FFI",
             "",
@@ -484,8 +485,9 @@
         self.ffi.verifier = Verifier(
             self.ffi, source, force_generic_engine=True, **kwargs)
         libfilename = '_'.join([self._module_name, libname])
-        self.ffi.verifier.make_library(
-            os.path.join(self._module_path, libfilename + _get_so_suffix()))
+        libfilepath = os.path.join(
+            self._module_path, libfilename + _get_so_suffix())
+        self.ffi.verifier.make_library(libfilepath)
         self._module_source += '\n'.join([
             "def load_%s():",
             "    from cffi.verifier import Verifier",
@@ -497,6 +499,7 @@
             "    return verifier._load_library()",
             "",
         ]) % (libname, libfilename)
+        self._built_files.append(libfilepath)
 
     def write_ffi_module(self):
         import os
@@ -505,9 +508,14 @@
         except OSError:
             pass
 
-        module_filename = self._module_name + '.py'
-        file = open(os.path.join(self._module_path, module_filename), 'w')
+        module_filepath = os.path.join(
+            self._module_path, self._module_name + '.py')
+        file = open(module_filepath, 'w')
         try:
             file.write(self._module_source)
         finally:
             file.close()
+        self._built_files.append(module_filepath)
+
+    def list_built_files(self):
+        return self._built_files
diff --git a/testing/test_makelib.py b/testing/test_makelib.py
--- a/testing/test_makelib.py
+++ b/testing/test_makelib.py
@@ -1,6 +1,7 @@
 import math
 import sys
 from cffi import FFIBuilder
+from cffi.verifier import _get_so_suffix
 
 
 def _clean_modules(tmpdir, module_name):
@@ -18,6 +19,11 @@
     builder.makelib('foo', '#include <math.h>')
     builder.write_ffi_module()
 
+    assert builder.list_built_files() == [
+        str(tmpdir.join('foo_ffi_foo' + _get_so_suffix())),
+        str(tmpdir.join('foo_ffi.py')),
+    ]
+
     sys.path.append(str(tmpdir))
     try:
         import foo_ffi
@@ -36,6 +42,10 @@
     builder.add_dlopen('foo', "m")
     builder.write_ffi_module()
 
+    assert builder.list_built_files() == [
+        str(tmpdir.join('foo_ffi.py')),
+    ]
+
     sys.path.append(str(tmpdir))
     try:
         import foo_ffi
@@ -55,6 +65,11 @@
     builder.add_dlopen('bar', "m")
     builder.write_ffi_module()
 
+    assert builder.list_built_files() == [
+        str(tmpdir.join('foo_ffi_foo' + _get_so_suffix())),
+        str(tmpdir.join('foo_ffi.py')),
+    ]
+
     sys.path.append(str(tmpdir))
     try:
         import foo_ffi

From noreply at buildbot.pypy.org  Sun Oct  6 13:21:50 2013
From: noreply at buildbot.pypy.org (jerith)
Date: Sun,  6 Oct 2013 13:21:50 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Move FFIBuilder into its own
	module.
Message-ID: <20131006112150.61B331C1190@cobra.cs.uni-duesseldorf.de>

Author: Jeremy Thurgood <firxen at gmail.com>
Branch: split-verify
Changeset: r1354:e5fe5874e20f
Date: 2013-10-06 13:21 +0200
http://bitbucket.org/cffi/cffi/changeset/e5fe5874e20f/

Log:	Move FFIBuilder into its own module.

diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -1,7 +1,8 @@
 __all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError',
-           'FFIError']
+           'FFIError', 'FFIBuilder']
 
-from .api import FFI, CDefError, FFIError, FFIBuilder
+from .api import FFI, CDefError, FFIError
+from .builder import FFIBuilder
 from .ffiplatform import VerificationError, VerificationMissing
 
 __version__ = "0.7.2"
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -448,74 +448,3 @@
         return None
     else:
         return ffi._get_cached_btype(tp)
-
-
-class FFIBuilder(object):
-    def __init__(self, module_name, module_path, backend=None):
-        self._module_name = module_name
-        self._module_path = module_path
-        self.ffi = FFI(backend=backend)
-        self._built_files = []
-        self._module_source = "\n".join([
-            "from cffi import FFI",
-            "",
-            "ffi = FFI()",
-            "",
-        ])
-
-    def cdef(self, csource, override=False):
-        self.ffi.cdef(csource, override=override)
-        self._module_source += "ffi.cdef(%r, override=%r)\n" % (
-            csource, override)
-
-    def add_dlopen(self, libname, name, flags=0):
-        lib = self.ffi.dlopen(name, flags=flags)
-        self._module_source += '\n'.join([
-            "def load_%s():",
-            "    return ffi.dlopen(%r, flags=%r)",
-            "",
-        ]) % (libname, name, flags)
-        return lib
-
-    def makelib(self, libname, source='', **kwargs):
-        # XXX: We use force_generic_engine here because vengine_cpy collects
-        #      types when it writes the source.
-        import os.path
-        from .verifier import Verifier, _get_so_suffix
-        self.ffi.verifier = Verifier(
-            self.ffi, source, force_generic_engine=True, **kwargs)
-        libfilename = '_'.join([self._module_name, libname])
-        libfilepath = os.path.join(
-            self._module_path, libfilename + _get_so_suffix())
-        self.ffi.verifier.make_library(libfilepath)
-        self._module_source += '\n'.join([
-            "def load_%s():",
-            "    from cffi.verifier import Verifier",
-            "    import os.path",
-            "    module_path = os.path.dirname(__file__)",
-            "    verifier = Verifier(",
-            "        ffi, None, module_path, %r, force_generic_engine=True)",
-            "    verifier._has_module = True",
-            "    return verifier._load_library()",
-            "",
-        ]) % (libname, libfilename)
-        self._built_files.append(libfilepath)
-
-    def write_ffi_module(self):
-        import os
-        try:
-            os.makedirs(self._module_path)
-        except OSError:
-            pass
-
-        module_filepath = os.path.join(
-            self._module_path, self._module_name + '.py')
-        file = open(module_filepath, 'w')
-        try:
-            file.write(self._module_source)
-        finally:
-            file.close()
-        self._built_files.append(module_filepath)
-
-    def list_built_files(self):
-        return self._built_files
diff --git a/cffi/builder.py b/cffi/builder.py
new file mode 100644
--- /dev/null
+++ b/cffi/builder.py
@@ -0,0 +1,72 @@
+from .api import FFI
+
+
+class FFIBuilder(object):
+    def __init__(self, module_name, module_path, backend=None):
+        self._module_name = module_name
+        self._module_path = module_path
+        self.ffi = FFI(backend=backend)
+        self._built_files = []
+        self._module_source = "\n".join([
+            "from cffi import FFI",
+            "",
+            "ffi = FFI()",
+            "",
+        ])
+
+    def cdef(self, csource, override=False):
+        self.ffi.cdef(csource, override=override)
+        self._module_source += "ffi.cdef(%r, override=%r)\n" % (
+            csource, override)
+
+    def add_dlopen(self, libname, name, flags=0):
+        lib = self.ffi.dlopen(name, flags=flags)
+        self._module_source += '\n'.join([
+            "def load_%s():",
+            "    return ffi.dlopen(%r, flags=%r)",
+            "",
+        ]) % (libname, name, flags)
+        return lib
+
+    def makelib(self, libname, source='', **kwargs):
+        # XXX: We use force_generic_engine here because vengine_cpy collects
+        #      types when it writes the source.
+        import os.path
+        from .verifier import Verifier, _get_so_suffix
+        self.ffi.verifier = Verifier(
+            self.ffi, source, force_generic_engine=True, **kwargs)
+        libfilename = '_'.join([self._module_name, libname])
+        libfilepath = os.path.join(
+            self._module_path, libfilename + _get_so_suffix())
+        self.ffi.verifier.make_library(libfilepath)
+        self._module_source += '\n'.join([
+            "def load_%s():",
+            "    from cffi.verifier import Verifier",
+            "    import os.path",
+            "    module_path = os.path.dirname(__file__)",
+            "    verifier = Verifier(",
+            "        ffi, None, module_path, %r, force_generic_engine=True)",
+            "    verifier._has_module = True",
+            "    return verifier._load_library()",
+            "",
+        ]) % (libname, libfilename)
+        self._built_files.append(libfilepath)
+
+    def write_ffi_module(self):
+        import os
+        try:
+            os.makedirs(self._module_path)
+        except OSError:
+            pass
+
+        module_filepath = os.path.join(
+            self._module_path, self._module_name + '.py')
+        file = open(module_filepath, 'w')
+        try:
+            file.write(self._module_source)
+        finally:
+            file.close()
+        self._built_files.append(module_filepath)
+
+    def list_built_files(self):
+        return self._built_files

From noreply at buildbot.pypy.org  Sun Oct  6 13:23:21 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  6 Oct 2013 13:23:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: Make the test more similar to
	pypy/module/_cffi_backend.
Message-ID: <20131006112321.0ABB01C1190@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67164:d0b269c45840
Date: 2013-10-06 13:22 +0200
http://bitbucket.org/pypy/pypy/changeset/d0b269c45840/

Log:	Make the test more similar to pypy/module/_cffi_backend. Crashes
	when a guard fails in the middle of the virtualized raw-malloc
	buffer.

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -381,6 +381,8 @@
             res = self.llinterp.eval_graph(ptr._obj.graph, args)
         else:
             res = ptr._obj._callable(*args)
+        if RESULT is lltype.Void:
+            return None
         return support.cast_result(RESULT, res)
 
     def _do_call(self, func, args_i, args_r, args_f, calldescr):
diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py
--- a/rpython/jit/metainterp/test/test_fficall.py
+++ b/rpython/jit/metainterp/test/test_fficall.py
@@ -86,15 +86,17 @@
             data = rffi.ptradd(exchange_buffer, ofs)
             rffi.cast(lltype.Ptr(TYPE), data)[0] = write_rvalue
 
-        def f():
+        def f(i):
             exbuf = lltype.malloc(rffi.CCHARP.TO, (len(avalues)+2) * 16,
-                                  flavor='raw', zero=True)
-            ofs = 16
+                                  flavor='raw')
+
+            targetptr = rffi.ptradd(exbuf, 16)
             for avalue in unroll_avalues:
                 TYPE = rffi.CArray(lltype.typeOf(avalue))
-                data = rffi.ptradd(exbuf, ofs)
-                rffi.cast(lltype.Ptr(TYPE), data)[0] = avalue
-                ofs += 16
+                if i == 9:    # a guard that can fail
+                    pass
+                rffi.cast(lltype.Ptr(TYPE), targetptr)[0] = avalue
+                targetptr = rffi.ptradd(targetptr, 16)
 
             jit_ffi_call(cif_description, func_addr, exbuf)
 
@@ -102,8 +104,7 @@
                 res = 654321
             else:
                 TYPE = rffi.CArray(lltype.typeOf(rvalue))
-                data = rffi.ptradd(exbuf, ofs)
-                res = rffi.cast(lltype.Ptr(TYPE), data)[0]
+                res = rffi.cast(lltype.Ptr(TYPE), targetptr)[0]
             lltype.free(exbuf, flavor='raw')
             if lltype.typeOf(res) is lltype.SingleFloat:
                 res = float(res)
@@ -117,9 +118,9 @@
             return res == rvalue
 
         with FakeFFI(fake_call_impl_any):
-            res = f()
+            res = f(-42)
             assert matching_result(res, rvalue)
-            res = self.interp_operations(f, [],
+            res = self.interp_operations(f, [-42],
                             supports_floats = supports_floats,
                           supports_longlong = supports_longlong,
                       supports_singlefloats = supports_singlefloats)
@@ -132,6 +133,19 @@
             self.check_operations_history(call_may_force=0,
                                           call_release_gil=expected_call_release_gil)
 
+        ##################################################
+        driver = jit.JitDriver(reds=['i'], greens=[])
+        def main():
+            i = 0
+            while 1:
+                driver.jit_merge_point(i=i)
+                res = f(i)
+                i += 1
+                if i == 12:
+                    return res
+        self.meta_interp(main, [])
+
+
     def test_simple_call_int(self):
         self._run([types.signed] * 2, types.signed, [456, 789], -42)
 

From noreply at buildbot.pypy.org  Sun Oct  6 13:48:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  6 Oct 2013 13:48:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: progressing on d0b269c45840: the issue
 is that VRawSliceValue()
Message-ID: <20131006114823.63AF21C0223@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67165:3e6b3a2d6bf7
Date: 2013-10-06 13:47 +0200
http://bitbucket.org/pypy/pypy/changeset/3e6b3a2d6bf7/

Log:	progressing on d0b269c45840: the issue is that VRawSliceValue()
	fails to override get_args_for_fail() and inherits the default one,
	which does nothing. This change shares some repeated code and shows
	the problem more directly.

diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -45,6 +45,15 @@
             return value
         return OptValue(self.force_box(optforce))
 
+    def get_args_for_fail(self, modifier):
+        # checks for recursion: it is False unless
+        # we have already seen the very same keybox
+        if self.box is None and not modifier.already_seen_virtual(self.keybox):
+            self._get_args_for_fail(modifier)
+
+    def _get_args_for_fail(self, modifier):
+        raise NotImplementedError("abstract base")
+
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
             return self._make_virtual(modifier)
@@ -193,16 +202,13 @@
             self._cached_sorted_fields = lst
         return lst
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            # checks for recursion: it is False unless
-            # we have already seen the very same keybox
-            lst = self._get_field_descr_list()
-            fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst]
-            modifier.register_virtual_fields(self.keybox, fieldboxes)
-            for ofs in lst:
-                fieldvalue = self._fields[ofs]
-                fieldvalue.get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        lst = self._get_field_descr_list()
+        fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst]
+        modifier.register_virtual_fields(self.keybox, fieldboxes)
+        for ofs in lst:
+            fieldvalue = self._fields[ofs]
+            fieldvalue.get_args_for_fail(modifier)
 
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
@@ -254,18 +260,15 @@
     def set_item_value(self, i, newval):
         raise NotImplementedError
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            # checks for recursion: it is False unless
-            # we have already seen the very same keybox
-            itemboxes = []
-            for i in range(self.getlength()):
-                itemvalue = self.get_item_value(i)
-                itemboxes.append(itemvalue.get_key_box())
-            modifier.register_virtual_fields(self.keybox, itemboxes)
-            for i in range(self.getlength()):
-                itemvalue = self.get_item_value(i)
-                itemvalue.get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        itemboxes = []
+        for i in range(self.getlength()):
+            itemvalue = self.get_item_value(i)
+            itemboxes.append(itemvalue.get_key_box())
+        modifier.register_virtual_fields(self.keybox, itemboxes)
+        for i in range(self.getlength()):
+            itemvalue = self.get_item_value(i)
+            itemvalue.get_args_for_fail(modifier)
 
 
 class VArrayValue(AbstractVArrayValue):
@@ -370,17 +373,16 @@
             descrs.append(item_descrs)
         return descrs
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            itemdescrs = self._get_list_of_descrs()
-            itemboxes = []
-            for i in range(len(self._items)):
-                for descr in itemdescrs[i]:
-                    itemboxes.append(self._items[i][descr].get_key_box())
-            modifier.register_virtual_fields(self.keybox, itemboxes)
-            for i in range(len(self._items)):
-                for descr in itemdescrs[i]:
-                    self._items[i][descr].get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        itemdescrs = self._get_list_of_descrs()
+        itemboxes = []
+        for i in range(len(self._items)):
+            for descr in itemdescrs[i]:
+                itemboxes.append(self._items[i][descr].get_key_box())
+        modifier.register_virtual_fields(self.keybox, itemboxes)
+        for i in range(len(self._items)):
+            for descr in itemdescrs[i]:
+                self._items[i][descr].get_args_for_fail(modifier)
 
     def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:

From noreply at buildbot.pypy.org  Sun Oct  6 14:37:49 2013
From: noreply at buildbot.pypy.org (jerith)
Date: Sun,  6 Oct 2013 14:37:49 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Better API for generated module.
Message-ID: <20131006123749.DE3981C0223@cobra.cs.uni-duesseldorf.de>

Author: Jeremy Thurgood <firxen at gmail.com>
Branch: split-verify
Changeset: r1355:b1c528ee24ae
Date: 2013-10-06 14:35 +0200
http://bitbucket.org/cffi/cffi/changeset/b1c528ee24ae/

Log:	Better API for generated module.

diff --git a/cffi/builder.py b/cffi/builder.py
--- a/cffi/builder.py
+++ b/cffi/builder.py
@@ -1,31 +1,136 @@
 from .api import FFI
 
 
+MODULE_BOILERPLATE = """
+#####                                                                  #####
+##### NOTE: This module is generated by cffi. DO NOT EDIT IT MANUALLY. #####
+#####                                                                  #####
+
+from functools import wraps
+from cffi import FFI
+
+
+_ffi = FFI()
+
+
+### The functions below are proxies for `_ffi` to make things more convenient.
+
+
+ at wraps(_ffi.typeof)
+def typeof(cdecl):
+    return _ffi.typeof(cdecl)
+
+
+ at wraps(_ffi.sizeof)
+def sizeof(cdecl):
+    return _ffi.sizeof(cdecl)
+
+
+ at wraps(_ffi.alignof)
+def alignof(cdecl):
+    return _ffi.alignof(cdecl)
+
+
+ at wraps(_ffi.offsetof)
+def offsetof(cdecl):
+    return _ffi.offsetof(cdecl)
+
+
+ at wraps(_ffi.new)
+def new(cdecl, init=None):
+    return _ffi.new(cdecl, init=init)
+
+
+ at wraps(_ffi.cast)
+def cast(cdecl, source):
+    return _ffi.cast(cdecl, source)
+
+
+ at wraps(_ffi.string)
+def string(cdecl, maxlen=-1):
+    return _ffi.string(cdecl, maxlen=-1)
+
+
+ at wraps(_ffi.buffer)
+def buffer(cdecl, maxlen=-1):
+    return _ffi.buffer(cdecl, maxlen=-1)
+
+
+ at wraps(_ffi.callback)
+def callback(cdecl, python_callable=None, error=None):
+    return _ffi.callback(cdecl, python_callable=python_callable, error=error)
+
+
+ at wraps(_ffi.getctype)
+def getctype(cdecl, replace_with=''):
+    return _ffi.getctype(cdecl, replace_with=replace_with)
+
+
+ at wraps(_ffi.gc)
+def gc(cdata, destructor):
+    return _ffi.gc(cdata, destructor)
+
+
+def _get_errno():
+    return _ffi.errno
+def _set_errno(errno):
+    _ffi.errno = errno
+errno = property(_get_errno, _set_errno, None,
+                 "the value of 'errno' from/to the C calls")
+
+
+ at wraps(_ffi.addressof)
+def addressof(cdata, field=None):
+    return _ffi.addressof(cdata, field=field)
+
+
+ at wraps(_ffi.new_handle)
+def new_handle(x):
+    return _ffi.new_handle(x)
+
+
+ at wraps(_ffi.from_handle)
+def from_handle(x):
+    return _ffi.from_handle(x)
+
+
+### The functions below are generated by cffi.
+"""
+
+
+DLOPEN_FUNC_TEMPLATE = """
+def load_%s():
+    return _ffi.dlopen(%r, flags=%r)
+"""
+
+
+MAKELIB_FUNC_TEMPLATE = """
+def load_%s():
+    import os.path
+    from cffi.verifier import Verifier
+    module_path = os.path.dirname(__file__)
+    verifier = Verifier(_ffi, None, module_path, %r, force_generic_engine=True)
+    verifier._has_module = True
+    return verifier._load_library()
+"""
+
+
 class FFIBuilder(object):
     def __init__(self, module_name, module_path, backend=None):
         self._module_name = module_name
         self._module_path = module_path
         self.ffi = FFI(backend=backend)
         self._built_files = []
-        self._module_source = "\n".join([
-            "from cffi import FFI",
-            "",
-            "ffi = FFI()",
-            "",
-        ])
+        self._module_source = MODULE_BOILERPLATE
 
     def cdef(self, csource, override=False):
         self.ffi.cdef(csource, override=override)
-        self._module_source += "ffi.cdef(%r, override=%r)\n" % (
+        self._module_source += "_ffi.cdef(%r, override=%r)\n" % (
             csource, override)
 
     def add_dlopen(self, libname, name, flags=0):
         lib = self.ffi.dlopen(name, flags=flags)
-        self._module_source += '\n'.join([
-            "def load_%s():",
-            "    return ffi.dlopen(%r, flags=%r)",
-            "",
-        ]) % (libname, name, flags)
+        self._module_source += DLOPEN_FUNC_TEMPLATE % (libname, name, flags)
         return lib
 
     def makelib(self, libname, source='', **kwargs):
@@ -39,18 +144,8 @@
         libfilepath = os.path.join(
             self._module_path, libfilename + _get_so_suffix())
         self.ffi.verifier.make_library(libfilepath)
-        self._module_source += '\n'.join([
-            "def load_%s():",
-            "    from cffi.verifier import Verifier",
-            "    import os.path",
-            "    module_path = os.path.dirname(__file__)",
-            "    verifier = Verifier(",
-            "        ffi, None, module_path, %r, force_generic_engine=True)",
-            "    verifier._has_module = True",
-            "    return verifier._load_library()",
-            "",
-        ]) % (libname, libfilename)
-        self._built_files.append(libfilepath)
+        self._module_source += MAKELIB_FUNC_TEMPLATE % (libname, libfilename)
+        self._built_files.append(libfilename + _get_so_suffix())
 
     def write_ffi_module(self):
         import os
@@ -59,14 +154,14 @@
         except OSError:
             pass
 
-        module_filepath = os.path.join(
-            self._module_path, self._module_name + '.py')
+        module_filename = self._module_name + '.py'
+        module_filepath = os.path.join(self._module_path, module_filename)
         file = open(module_filepath, 'w')
         try:
             file.write(self._module_source)
         finally:
             file.close()
-        self._built_files.append(module_filepath)
+        self._built_files.append(module_filename)
 
     def list_built_files(self):
         return self._built_files
diff --git a/testing/test_makelib.py b/testing/test_makelib.py
--- a/testing/test_makelib.py
+++ b/testing/test_makelib.py
@@ -20,8 +20,8 @@
     builder.write_ffi_module()
 
     assert builder.list_built_files() == [
-        str(tmpdir.join('foo_ffi_foo' + _get_so_suffix())),
-        str(tmpdir.join('foo_ffi.py')),
+        'foo_ffi_foo' + _get_so_suffix(),
+        'foo_ffi.py',
     ]
 
     sys.path.append(str(tmpdir))
@@ -43,7 +43,7 @@
     builder.write_ffi_module()
 
     assert builder.list_built_files() == [
-        str(tmpdir.join('foo_ffi.py')),
+        'foo_ffi.py',
     ]
 
     sys.path.append(str(tmpdir))
@@ -66,8 +66,8 @@
     builder.write_ffi_module()
 
     assert builder.list_built_files() == [
-        str(tmpdir.join('foo_ffi_foo' + _get_so_suffix())),
-        str(tmpdir.join('foo_ffi.py')),
+        'foo_ffi_foo' + _get_so_suffix(),
+        'foo_ffi.py',
     ]
 
     sys.path.append(str(tmpdir))

From noreply at buildbot.pypy.org  Sun Oct  6 14:42:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  6 Oct 2013 14:42:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add the missing VRawSliceInfo,
	improve the test, fix.
Message-ID: <20131006124219.E56F21C01B0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67166:1cd66e3ec8aa
Date: 2013-10-06 14:41 +0200
http://bitbucket.org/pypy/pypy/changeset/1cd66e3ec8aa/

Log:	Add the missing VRawSliceInfo, improve the test, fix.

diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -483,6 +483,15 @@
     def getitem_raw(self, offset, length, descr):
         return self.rawbuffer_value.getitem_raw(self.offset+offset, length, descr)
 
+    def _get_args_for_fail(self, modifier):
+        box = self.rawbuffer_value.get_key_box()
+        modifier.register_virtual_fields(self.keybox, [box])
+        self.rawbuffer_value.get_args_for_fail(modifier)
+
+    def _make_virtual(self, modifier):
+        return modifier.make_vrawslice(self.offset)
+
+
 class OptVirtualize(optimizer.Optimization):
     "Virtualize objects until they escape."
 
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -284,7 +284,10 @@
         return VArrayStructInfo(arraydescr, fielddescrs)
 
     def make_vrawbuffer(self, size, offsets, descrs):
-        return VRawBufferStateInfo(size, offsets, descrs)
+        return VRawBufferInfo(size, offsets, descrs)
+
+    def make_vrawslice(self, offset):
+        return VRawSliceInfo(offset)
 
     def make_vstrplain(self, is_unicode=False):
         if is_unicode:
@@ -554,10 +557,13 @@
             debug_print("\t\t", str(untag(i)))
 
 
-class VRawBufferStateInfo(AbstractVirtualInfo):
+class VAbstractRawInfo(AbstractVirtualInfo):
     kind = INT
     is_about_raw = True
 
+
+class VRawBufferInfo(VAbstractRawInfo):
+
     def __init__(self, size, offsets, descrs):
         self.size = size
         self.offsets = offsets
@@ -580,6 +586,25 @@
             debug_print("\t\t", str(untag(i)))
 
 
+class VRawSliceInfo(VAbstractRawInfo):
+
+    def __init__(self, offset):
+        self.offset = offset
+
+    @specialize.argtype(1)
+    def allocate_int(self, decoder, index):
+        assert len(self.fieldnums) == 1
+        base_buffer = decoder.decode_int(self.fieldnums[0])
+        buffer = decoder.int_add_const(base_buffer, self.offset)
+        decoder.virtuals_cache.set_int(index, buffer)
+        return buffer
+
+    def debug_prints(self):
+        debug_print("\tvrawsliceinfo", " at ",  compute_unique_id(self))
+        for i in self.fieldnums:
+            debug_print("\t\t", str(untag(i)))
+
+
 class VArrayStructInfo(AbstractVirtualInfo):
     def __init__(self, arraydescr, fielddescrs):
         self.arraydescr = arraydescr
@@ -784,7 +809,7 @@
         if not v:
             v = self.rd_virtuals[index]
             ll_assert(bool(v), "resume.py: null rd_virtuals[index]")
-            assert v.is_about_raw and isinstance(v, VRawBufferStateInfo)
+            assert v.is_about_raw and isinstance(v, VAbstractRawInfo)
             v = v.allocate_int(self, index)
             ll_assert(v == self.virtuals_cache.get_int(index), "resume.py: bad cache")
         return v
@@ -1117,6 +1142,10 @@
     def write_a_float(self, index, box):
         self.boxes_f[index] = box
 
+    def int_add_const(self, intbox, offset):
+        return self.metainterp.execute_and_record(rop.INT_ADD, None, intbox,
+                                                  ConstInt(offset))
+
 # ---------- when resuming for blackholing, get direct values ----------
 
 def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage,
@@ -1408,6 +1437,9 @@
     def write_a_float(self, index, float):
         self.blackholeinterp.setarg_f(index, float)
 
+    def int_add_const(self, base, offset):
+        return base + offset
+
 # ____________________________________________________________
 
 def dump_storage(storage, liveboxes):
diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py
--- a/rpython/jit/metainterp/test/test_fficall.py
+++ b/rpython/jit/metainterp/test/test_fficall.py
@@ -93,7 +93,7 @@
             targetptr = rffi.ptradd(exbuf, 16)
             for avalue in unroll_avalues:
                 TYPE = rffi.CArray(lltype.typeOf(avalue))
-                if i == 9:    # a guard that can fail
+                if i >= 9:    # a guard that can fail
                     pass
                 rffi.cast(lltype.Ptr(TYPE), targetptr)[0] = avalue
                 targetptr = rffi.ptradd(targetptr, 16)

From noreply at buildbot.pypy.org  Sun Oct  6 15:06:00 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Sun,  6 Oct 2013 15:06:00 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Add cffi.packaging module which
 has everything you need for your setup.py
Message-ID: <20131006130600.724341C01B0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1356:5e2da61b8aa5
Date: 2013-10-06 15:02 +0200
http://bitbucket.org/cffi/cffi/changeset/5e2da61b8aa5/

Log:	Add cffi.packaging module which has everything you need for your
	setup.py

diff --git a/cffi/packaging.py b/cffi/packaging.py
new file mode 100644
--- /dev/null
+++ b/cffi/packaging.py
@@ -0,0 +1,21 @@
+from distutils.command.build_ext import build_ext as _build_ext
+from distutils.core import Extension
+import os
+
+
+class FFIExtension(Extension):
+    def __init__(self, ffi_builder):
+        self.ffi_builder = ffi_builder
+        Extension.__init__(self, '<cffi extension>', [])
+
+
+class build_ext(_build_ext):
+    def build_extension(self, ext):
+        if isinstance(ext, FFIExtension):
+            files = ext.ffi_builder(self.build_temp)
+            for name in files:
+                self.copy_file(
+                    os.path.join(self.build_temp, name),
+                    os.path.join(self.build_lib, name))
+        else:
+            super(build_ext, self).build_extension(ext)

From noreply at buildbot.pypy.org  Sun Oct  6 15:21:44 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  6 Oct 2013 15:21:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test fix
Message-ID: <20131006132144.131201C0223@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67167:551d3b942fd0
Date: 2013-10-06 15:18 +0200
http://bitbucket.org/pypy/pypy/changeset/551d3b942fd0/

Log:	Test fix

diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py
--- a/rpython/jit/metainterp/test/test_fficall.py
+++ b/rpython/jit/metainterp/test/test_fficall.py
@@ -133,17 +133,17 @@
             self.check_operations_history(call_may_force=0,
                                           call_release_gil=expected_call_release_gil)
 
-        ##################################################
-        driver = jit.JitDriver(reds=['i'], greens=[])
-        def main():
-            i = 0
-            while 1:
-                driver.jit_merge_point(i=i)
-                res = f(i)
-                i += 1
-                if i == 12:
-                    return res
-        self.meta_interp(main, [])
+            ##################################################
+            driver = jit.JitDriver(reds=['i'], greens=[])
+            def main():
+                i = 0
+                while 1:
+                    driver.jit_merge_point(i=i)
+                    res = f(i)
+                    i += 1
+                    if i == 12:
+                        return res
+            self.meta_interp(main, [])
 
 
     def test_simple_call_int(self):

From noreply at buildbot.pypy.org  Sun Oct  6 15:31:43 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  6 Oct 2013 15:31:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: A test crashing on a recent pypy,
 which I hope is fixed now
Message-ID: <20131006133143.80D641C0223@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67168:b355653b712a
Date: 2013-10-06 15:31 +0200
http://bitbucket.org/pypy/pypy/changeset/b355653b712a/

Log:	A test crashing on a recent pypy, which I hope is fixed now by
	1cd66e3ec8aa.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -277,3 +277,28 @@
             f1 = call_release_gil(..., descr=<Calli 4 ii EF=6 OS=62>)
             ...
         """)
+
+    def test__cffi_bug1(self):
+        from rpython.rlib.test.test_clibffi import get_libm_name
+        def main(libm_name):
+            try:
+                import _cffi_backend
+            except ImportError:
+                sys.stderr.write('SKIP: cannot import _cffi_backend\n')
+                return 0
+
+            libm = _cffi_backend.load_library(libm_name)
+            BDouble = _cffi_backend.new_primitive_type("double")
+            BSin = _cffi_backend.new_function_type([BDouble], BDouble)
+            sin = libm.load_function(BSin, 'sin')
+
+            def f(*args):
+                for i in range(300):
+                    sin(*args)
+
+            f(1.0)
+            f(1)
+        #
+        libm_name = get_libm_name(sys.platform)
+        log = self.run(main, [libm_name])
+        # assert did not crash

From noreply at buildbot.pypy.org  Sun Oct  6 15:36:37 2013
From: noreply at buildbot.pypy.org (jerith)
Date: Sun,  6 Oct 2013 15:36:37 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Fix (and test) ffi proxy functions.
Message-ID: <20131006133637.05A4A1C1190@cobra.cs.uni-duesseldorf.de>

Author: Jeremy Thurgood <firxen at gmail.com>
Branch: split-verify
Changeset: r1357:9f5e04349245
Date: 2013-10-06 15:36 +0200
http://bitbucket.org/cffi/cffi/changeset/9f5e04349245/

Log:	Fix (and test) ffi proxy functions.

diff --git a/cffi/builder.py b/cffi/builder.py
--- a/cffi/builder.py
+++ b/cffi/builder.py
@@ -13,85 +13,34 @@
 _ffi = FFI()
 
 
-### The functions below are proxies for `_ffi` to make things more convenient.
+### Proxy `_ffi` methods to make things more convenient.
 
 
- at wraps(_ffi.typeof)
-def typeof(cdecl):
-    return _ffi.typeof(cdecl)
+typeof = _ffi.typeof
+sizeof = _ffi.sizeof
+alignof = _ffi.alignof
+offsetof = _ffi.offsetof
+new = _ffi.new
+cast = _ffi.cast
+string = _ffi.string
+buffer = _ffi.buffer
+callback = _ffi.callback
+getctype = _ffi.getctype
+gc = _ffi.gc
 
 
- at wraps(_ffi.sizeof)
-def sizeof(cdecl):
-    return _ffi.sizeof(cdecl)
+# Can't have properties on modules. :-(
 
+def get_errno():
+    return _ffi.errno
 
- at wraps(_ffi.alignof)
-def alignof(cdecl):
-    return _ffi.alignof(cdecl)
+def set_errno(errno):
+    _ffi.errno = errno
 
 
- at wraps(_ffi.offsetof)
-def offsetof(cdecl):
-    return _ffi.offsetof(cdecl)
-
-
- at wraps(_ffi.new)
-def new(cdecl, init=None):
-    return _ffi.new(cdecl, init=init)
-
-
- at wraps(_ffi.cast)
-def cast(cdecl, source):
-    return _ffi.cast(cdecl, source)
-
-
- at wraps(_ffi.string)
-def string(cdecl, maxlen=-1):
-    return _ffi.string(cdecl, maxlen=-1)
-
-
- at wraps(_ffi.buffer)
-def buffer(cdecl, maxlen=-1):
-    return _ffi.buffer(cdecl, maxlen=-1)
-
-
- at wraps(_ffi.callback)
-def callback(cdecl, python_callable=None, error=None):
-    return _ffi.callback(cdecl, python_callable=python_callable, error=error)
-
-
- at wraps(_ffi.getctype)
-def getctype(cdecl, replace_with=''):
-    return _ffi.getctype(cdecl, replace_with=replace_with)
-
-
- at wraps(_ffi.gc)
-def gc(cdata, destructor):
-    return _ffi.gc(cdata, destructor)
-
-
-def _get_errno():
-    return _ffi.errno
-def _set_errno(errno):
-    _ffi.errno = errno
-errno = property(_get_errno, _set_errno, None,
-                 "the value of 'errno' from/to the C calls")
-
-
- at wraps(_ffi.addressof)
-def addressof(cdata, field=None):
-    return _ffi.addressof(cdata, field=field)
-
-
- at wraps(_ffi.new_handle)
-def new_handle(x):
-    return _ffi.new_handle(x)
-
-
- at wraps(_ffi.from_handle)
-def from_handle(x):
-    return _ffi.from_handle(x)
+addressof = _ffi.addressof
+new_handle = _ffi.new_handle
+from_handle = _ffi.from_handle
 
 
 ### The functions below are generated by cffi.
diff --git a/testing/test_makelib.py b/testing/test_makelib.py
--- a/testing/test_makelib.py
+++ b/testing/test_makelib.py
@@ -80,3 +80,73 @@
     assert lib_foo.sin(12.3) == math.sin(12.3)
     lib_bar = foo_ffi.load_bar()
     assert lib_bar.sin(12.3) == math.sin(12.3)
+
+
+def test_ffi_module_functions(tmpdir):
+    builder = FFIBuilder("foo_ffi", str(tmpdir))
+    builder.cdef("""
+        double sin(double x);
+    """)
+    builder.makelib('foo', '#include <math.h>')
+    builder.write_ffi_module()
+
+    sys.path.append(str(tmpdir))
+    try:
+        import foo_ffi
+    finally:
+        _clean_modules(tmpdir, 'foo_ffi')
+
+    assert foo_ffi.typeof == foo_ffi._ffi.typeof
+    assert foo_ffi.sizeof == foo_ffi._ffi.sizeof
+    assert foo_ffi.alignof == foo_ffi._ffi.alignof
+    assert foo_ffi.offsetof == foo_ffi._ffi.offsetof
+    assert foo_ffi.new == foo_ffi._ffi.new
+    assert foo_ffi.cast == foo_ffi._ffi.cast
+    assert foo_ffi.string == foo_ffi._ffi.string
+    assert foo_ffi.buffer == foo_ffi._ffi.buffer
+    assert foo_ffi.callback == foo_ffi._ffi.callback
+    assert foo_ffi.getctype == foo_ffi._ffi.getctype
+    assert foo_ffi.gc == foo_ffi._ffi.gc
+
+    foo_ffi.set_errno(7)
+    assert foo_ffi.get_errno() == 7
+
+    assert foo_ffi.addressof == foo_ffi._ffi.addressof
+    assert foo_ffi.new_handle == foo_ffi._ffi.new_handle
+    assert foo_ffi.from_handle == foo_ffi._ffi.from_handle
+
+
+def test_ffi_do_some_stuff(tmpdir):
+    builder = FFIBuilder("foo_ffi", str(tmpdir))
+    builder.cdef("""
+        struct foo_s { int x; int y; };
+        int grid_distance(struct foo_s offset);
+    """)
+    builder.makelib('foo', """
+        struct foo_s { int x; int y; };
+        int grid_distance(struct foo_s offset) {
+            return offset.x + offset.y;
+        }
+    """)
+    builder.write_ffi_module()
+
+    sys.path.append(str(tmpdir))
+    try:
+        import foo_ffi
+    finally:
+        _clean_modules(tmpdir, 'foo_ffi')
+
+    my_struct = foo_ffi.new('struct foo_s *', {'x': 1, 'y': 2})
+    assert foo_ffi.typeof(my_struct) == foo_ffi.typeof("struct foo_s *")
+    assert foo_ffi.sizeof('struct foo_s') == 2 * foo_ffi.sizeof('int')
+    assert foo_ffi.alignof('struct foo_s') == foo_ffi.sizeof('int')
+    assert foo_ffi.typeof(foo_ffi.cast('long', 42)) == foo_ffi.typeof('long')
+    assert foo_ffi.string(foo_ffi.new('char *', b"\x00")) == b""
+
+    def cb(n):
+        return n + 1
+    f = foo_ffi.callback("int(*)(int)", cb)
+    assert f(1) == 2
+
+    lib = foo_ffi.load_foo()
+    assert lib.grid_distance(my_struct[0]) == 3

From noreply at buildbot.pypy.org  Sun Oct  6 18:09:42 2013
From: noreply at buildbot.pypy.org (jerith)
Date: Sun,  6 Oct 2013 18:09:42 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: I think this is the most horrible
 thing I have written in months, but... pickle all the _declarations.
Message-ID: <20131006160942.A8C6B1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Jeremy Thurgood <firxen at gmail.com>
Branch: split-verify
Changeset: r1358:1e40024fb288
Date: 2013-10-06 18:09 +0200
http://bitbucket.org/cffi/cffi/changeset/1e40024fb288/

Log:	I think this is the most horrible thing I have written in months,
	but... pickle all the _declarations.

diff --git a/cffi/builder.py b/cffi/builder.py
--- a/cffi/builder.py
+++ b/cffi/builder.py
@@ -1,3 +1,5 @@
+import pickle
+
 from .api import FFI
 
 
@@ -6,7 +8,7 @@
 ##### NOTE: This module is generated by cffi. DO NOT EDIT IT MANUALLY. #####
 #####                                                                  #####
 
-from functools import wraps
+import pickle
 from cffi import FFI
 
 
@@ -74,8 +76,6 @@
 
     def cdef(self, csource, override=False):
         self.ffi.cdef(csource, override=override)
-        self._module_source += "_ffi.cdef(%r, override=%r)\n" % (
-            csource, override)
 
     def add_dlopen(self, libname, name, flags=0):
         lib = self.ffi.dlopen(name, flags=flags)
@@ -97,6 +97,9 @@
         self._built_files.append(libfilename + _get_so_suffix())
 
     def write_ffi_module(self):
+        self._module_source += (
+            "_ffi._parser._declarations = pickle.loads(%r)" %
+            pickle.dumps(self.ffi._parser._declarations, 2))
         import os
         try:
             os.makedirs(self._module_path)

From noreply at buildbot.pypy.org  Sun Oct  6 19:28:23 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Sun,  6 Oct 2013 19:28:23 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: PEP-8 order the test_zintegration
	imports
Message-ID: <20131006172823.E90CB1C0223@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1359:34e95a90b3e1
Date: 2013-10-06 18:59 +0200
http://bitbucket.org/cffi/cffi/changeset/34e95a90b3e1/

Log:	PEP-8 order the test_zintegration imports

diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py
--- a/testing/test_zintegration.py
+++ b/testing/test_zintegration.py
@@ -1,8 +1,13 @@
-import py, os, sys, shutil
 import imp
+import os
+import shutil
 import subprocess
+import sys
+
+import py
 from testing.udir import udir
 
+
 def create_venv(name):
     tmpdir = udir.join(name)
     try:

From noreply at buildbot.pypy.org  Sun Oct  6 19:28:25 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Sun,  6 Oct 2013 19:28:25 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Add verifier_distutils_module,
 so that we can transition all the other zintegration tests to
 FFIBuilder
Message-ID: <20131006172825.05D691C1152@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1360:e898f9567c25
Date: 2013-10-06 19:04 +0200
http://bitbucket.org/cffi/cffi/changeset/e898f9567c25/

Log:	Add verifier_distutils_module, so that we can transition all the
	other zintegration tests to FFIBuilder

diff --git a/testing/snippets/verifier_distutils_module/setup.py b/testing/snippets/verifier_distutils_module/setup.py
new file mode 100644
--- /dev/null
+++ b/testing/snippets/verifier_distutils_module/setup.py
@@ -0,0 +1,7 @@
+
+from distutils.core import setup
+import snip_basic_verify
+
+setup(
+    py_modules=['snip_basic_verify'],
+    ext_modules=[snip_basic_verify.ffi.verifier.get_extension()])
diff --git a/testing/snippets/verifier_distutils_module/snip_basic_verify.py b/testing/snippets/verifier_distutils_module/snip_basic_verify.py
new file mode 100644
--- /dev/null
+++ b/testing/snippets/verifier_distutils_module/snip_basic_verify.py
@@ -0,0 +1,17 @@
+
+from cffi import FFI
+import sys
+
+ffi = FFI()
+ffi.cdef("""     // some declarations from the man page
+    struct passwd {
+        char *pw_name;
+        ...;
+    };
+    struct passwd *getpwuid(int uid);
+""")
+C = ffi.verify("""   // passed to the real C compiler
+#include <sys/types.h>
+#include <pwd.h>
+""", libraries=[],    # or a list of libraries to link with
+     force_generic_engine=hasattr(sys, '_force_generic_engine_'))
diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py
--- a/testing/test_zintegration.py
+++ b/testing/test_zintegration.py
@@ -83,6 +83,13 @@
     assert snip_infrastructure.func() == 42
     ''')
 
+def test_verifier_distutils_module():
+    run_setup_and_program("verifier_distutils_module", '''
+    import snip_basic_verify
+    p = snip_basic_verify.C.getpwuid(0)
+    assert snip_basic_verify.ffi.string(p.pw_name) == b"root"
+    ''')
+
 def test_distutils_module():
     run_setup_and_program("distutils_module", '''
     import snip_basic_verify

From noreply at buildbot.pypy.org  Sun Oct  6 19:28:26 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Sun,  6 Oct 2013 19:28:26 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Avoid setting force_generic_engine
	twice
Message-ID: <20131006172826.0874A1C0223@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1361:7e2fb851e5b5
Date: 2013-10-06 19:19 +0200
http://bitbucket.org/cffi/cffi/changeset/7e2fb851e5b5/

Log:	Avoid setting force_generic_engine twice

diff --git a/cffi/builder.py b/cffi/builder.py
--- a/cffi/builder.py
+++ b/cffi/builder.py
@@ -85,10 +85,10 @@
     def makelib(self, libname, source='', **kwargs):
         # XXX: We use force_generic_engine here because vengine_cpy collects
         #      types when it writes the source.
+        kwargs['force_generic_engine'] = True
         import os.path
         from .verifier import Verifier, _get_so_suffix
-        self.ffi.verifier = Verifier(
-            self.ffi, source, force_generic_engine=True, **kwargs)
+        self.ffi.verifier = Verifier(self.ffi, source, **kwargs)
         libfilename = '_'.join([self._module_name, libname])
         libfilepath = os.path.join(
             self._module_path, libfilename + _get_so_suffix())

From noreply at buildbot.pypy.org  Sun Oct  6 19:28:27 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Sun,  6 Oct 2013 19:28:27 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Create lib_dir if it doesn't exist
Message-ID: <20131006172827.064B41C0223@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1362:70f2aa466b82
Date: 2013-10-06 19:21 +0200
http://bitbucket.org/cffi/cffi/changeset/70f2aa466b82/

Log:	Create lib_dir if it doesn't exist

diff --git a/cffi/packaging.py b/cffi/packaging.py
--- a/cffi/packaging.py
+++ b/cffi/packaging.py
@@ -13,6 +13,8 @@
     def build_extension(self, ext):
         if isinstance(ext, FFIExtension):
             files = ext.ffi_builder(self.build_temp)
+            if not os.path.isdir(self.build_lib):
+                os.mkdir(self.build_lib)
             for name in files:
                 self.copy_file(
                     os.path.join(self.build_temp, name),

From noreply at buildbot.pypy.org  Sun Oct  6 19:28:28 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Sun,  6 Oct 2013 19:28:28 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Port distutils_module to FFIBuilder
Message-ID: <20131006172828.0BED41C0223@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1363:a983c3e85b1c
Date: 2013-10-06 19:28 +0200
http://bitbucket.org/cffi/cffi/changeset/a983c3e85b1c/

Log:	Port distutils_module to FFIBuilder

diff --git a/testing/snippets/distutils_module/ffibuilder.py b/testing/snippets/distutils_module/ffibuilder.py
new file mode 100644
--- /dev/null
+++ b/testing/snippets/distutils_module/ffibuilder.py
@@ -0,0 +1,21 @@
+import sys
+
+from cffi import FFIBuilder
+
+
+def build_ffi(path):
+    builder = FFIBuilder('snip_basic_module', path)
+    builder.cdef("""     // some declarations from the man page
+    struct passwd {
+        char *pw_name;
+        ...;
+    };
+    struct passwd *getpwuid(int uid);
+    """)
+    builder.makelib('passwd', """   // passed to the real C compiler
+    #include <sys/types.h>
+    #include <pwd.h>
+    """, libraries=[],  # or a list of libraries to link with
+         force_generic_engine=hasattr(sys, '_force_generic_engine_'))
+    builder.write_ffi_module()
+    return builder.list_built_files()
diff --git a/testing/snippets/distutils_module/setup.py b/testing/snippets/distutils_module/setup.py
--- a/testing/snippets/distutils_module/setup.py
+++ b/testing/snippets/distutils_module/setup.py
@@ -1,7 +1,10 @@
+from distutils.core import setup
 
-from distutils.core import setup
-import snip_basic_verify
+from cffi.packaging import FFIExtension, build_ext
+
+import ffibuilder
 
 setup(
-    py_modules=['snip_basic_verify'],
-    ext_modules=[snip_basic_verify.ffi.verifier.get_extension()])
+    ext_modules=[FFIExtension(ffibuilder.build_ffi)],
+    cmdclass={'build_ext': build_ext},
+)
diff --git a/testing/snippets/distutils_module/snip_basic_verify.py b/testing/snippets/distutils_module/snip_basic_verify.py
deleted file mode 100644
--- a/testing/snippets/distutils_module/snip_basic_verify.py
+++ /dev/null
@@ -1,17 +0,0 @@
-
-from cffi import FFI
-import sys
-
-ffi = FFI()
-ffi.cdef("""     // some declarations from the man page
-    struct passwd {
-        char *pw_name;
-        ...;
-    };
-    struct passwd *getpwuid(int uid);
-""")
-C = ffi.verify("""   // passed to the real C compiler
-#include <sys/types.h>
-#include <pwd.h>
-""", libraries=[],    # or a list of libraries to link with
-     force_generic_engine=hasattr(sys, '_force_generic_engine_'))
diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py
--- a/testing/test_zintegration.py
+++ b/testing/test_zintegration.py
@@ -92,9 +92,10 @@
 
 def test_distutils_module():
     run_setup_and_program("distutils_module", '''
-    import snip_basic_verify
-    p = snip_basic_verify.C.getpwuid(0)
-    assert snip_basic_verify.ffi.string(p.pw_name) == b"root"
+    import snip_basic_module
+    lib = snip_basic_module.load_passwd()
+    p = lib.getpwuid(0)
+    assert snip_basic_module.string(p.pw_name) == b"root"
     ''')
 
 def test_distutils_package_1():

From noreply at buildbot.pypy.org  Sun Oct  6 21:47:10 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  6 Oct 2013 21:47:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: implement, test more of numpy c api
Message-ID: <20131006194710.4C0AC1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67169:0be2466b3263
Date: 2013-10-06 22:46 +0300
http://bitbucket.org/pypy/pypy/changeset/0be2466b3263/

Log:	implement, test more of numpy c api

diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -149,14 +149,17 @@
          only used if the array is constructed that way. Almost always this
          parameter is NULL.
     """
-    if min_depth !=0 or max_depth != 0:
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            '_PyArray_FromAny called with not-implemented min_dpeth or max_depth argument'))
     if requirements not in (0, NPY_DEFAULT):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             '_PyArray_FromAny called with not-implemented requirements argument'))
     w_array = array(space, w_obj, w_dtype=w_dtype, copy=False)
-    if w_array.is_scalar():
+    if min_depth !=0 and len(w_array.get_shape()) < min_depth:
+        raise OperationError(space.w_ValueError, space.wrap(
+            'object of too small depth for desired array'))
+    elif max_depth !=0 and len(w_array.get_shape()) > max_depth:
+        raise OperationError(space.w_ValueError, space.wrap(
+            'object of too deep for desired array'))
+    elif w_array.is_scalar():
         # since PyArray_DATA() fails on scalars, create a 1D array and set empty
         # shape. So the following combination works for *reading* scalars:
         #     PyObject *arr = PyArray_FromAny(obj);
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -90,15 +90,16 @@
     def test_FromAny(self, space, api):
         a = array(space, [10, 5, 3])
         assert api._PyArray_FromAny(a, NULL, 0, 0, 0, NULL) is a
-        self.raises(space, api, NotImplementedError, api._PyArray_FromAny,
-                    a, NULL, 0, 3, 0, NULL)
+        assert api._PyArray_FromAny(a, NULL, 1, 4, 0, NULL) is a
+        self.raises(space, api, ValueError, api._PyArray_FromAny,
+                    a, NULL, 4, 5, 0, NULL)
 
     def test_FromObject(self, space, api):
         a = array(space, [10, 5, 3])
         assert api._PyArray_FromObject(a, a.get_dtype().num, 0, 0) is a
-        exc = self.raises(space, api, NotImplementedError, api._PyArray_FromObject,
-                    a, 11, 0, 3)
-        assert exc.errorstr(space).find('FromObject') >= 0
+        exc = self.raises(space, api, ValueError, api._PyArray_FromObject,
+                    a, 11, 4, 5)
+        assert exc.errorstr(space).find('desired') >= 0
 
     def test_list_from_fixedptr(self, space, api):
         A = lltype.GcArray(lltype.Float)

From noreply at buildbot.pypy.org  Mon Oct  7 01:10:22 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Mon,  7 Oct 2013 01:10:22 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: More whitespace
Message-ID: <20131006231022.8F23E1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1364:b3310f1eaa75
Date: 2013-10-06 23:52 +0200
http://bitbucket.org/cffi/cffi/changeset/b3310f1eaa75/

Log:	More whitespace

diff --git a/cffi/builder.py b/cffi/builder.py
--- a/cffi/builder.py
+++ b/cffi/builder.py
@@ -98,7 +98,7 @@
 
     def write_ffi_module(self):
         self._module_source += (
-            "_ffi._parser._declarations = pickle.loads(%r)" %
+            "_ffi._parser._declarations = pickle.loads(%r)\n" %
             pickle.dumps(self.ffi._parser._declarations, 2))
         import os
         try:

From noreply at buildbot.pypy.org  Mon Oct  7 01:10:23 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Mon,  7 Oct 2013 01:10:23 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Allow specifying a parent package
Message-ID: <20131006231023.9699F1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1365:32908a1690ea
Date: 2013-10-07 01:03 +0200
http://bitbucket.org/cffi/cffi/changeset/32908a1690ea/

Log:	Allow specifying a parent package

diff --git a/cffi/builder.py b/cffi/builder.py
--- a/cffi/builder.py
+++ b/cffi/builder.py
@@ -1,3 +1,4 @@
+import os
 import pickle
 
 from .api import FFI
@@ -67,13 +68,22 @@
 
 
 class FFIBuilder(object):
-    def __init__(self, module_name, module_path, backend=None):
+    def __init__(self, module_name, build_path, backend=None):
+        module_package = ''
+        if '.' in module_name:
+            module_package, module_name = module_name.rsplit('.', 1)
+        self._module_package = module_package
         self._module_name = module_name
-        self._module_path = module_path
+        self._build_path = build_path
         self.ffi = FFI(backend=backend)
         self._built_files = []
         self._module_source = MODULE_BOILERPLATE
 
+    def _filename(self, name, suffix='.py'):
+        parts = self._module_package.split('.')
+        parts.append(name + suffix)
+        return os.path.join(*parts)
+
     def cdef(self, csource, override=False):
         self.ffi.cdef(csource, override=override)
 
@@ -86,34 +96,32 @@
         # XXX: We use force_generic_engine here because vengine_cpy collects
         #      types when it writes the source.
         kwargs['force_generic_engine'] = True
-        import os.path
         from .verifier import Verifier, _get_so_suffix
         self.ffi.verifier = Verifier(self.ffi, source, **kwargs)
-        libfilename = '_'.join([self._module_name, libname])
-        libfilepath = os.path.join(
-            self._module_path, libfilename + _get_so_suffix())
-        self.ffi.verifier.make_library(libfilepath)
-        self._module_source += MAKELIB_FUNC_TEMPLATE % (libname, libfilename)
-        self._built_files.append(libfilename + _get_so_suffix())
+        barefilename = '_'.join([self._module_name, libname])
+        libfile_path = self._filename(barefilename, _get_so_suffix())
+        libfile_build_path = os.path.join(self._build_path, libfile_path)
+        self.ffi.verifier.make_library(libfile_build_path)
+        self._module_source += MAKELIB_FUNC_TEMPLATE % (libname, barefilename)
+        self._built_files.append(libfile_path)
 
     def write_ffi_module(self):
         self._module_source += (
             "_ffi._parser._declarations = pickle.loads(%r)\n" %
             pickle.dumps(self.ffi._parser._declarations, 2))
-        import os
         try:
-            os.makedirs(self._module_path)
+            os.makedirs(self._build_path)
         except OSError:
             pass
 
-        module_filename = self._module_name + '.py'
-        module_filepath = os.path.join(self._module_path, module_filename)
-        file = open(module_filepath, 'w')
+        module_path = self._filename(self._module_name)
+        module_build_path = os.path.join(self._build_path, module_path)
+        file = open(module_build_path, 'w')
         try:
             file.write(self._module_source)
         finally:
             file.close()
-        self._built_files.append(module_filename)
+        self._built_files.append(module_path)
 
     def list_built_files(self):
         return self._built_files

From noreply at buildbot.pypy.org  Mon Oct  7 01:10:24 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Mon,  7 Oct 2013 01:10:24 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Port test_distutils_package_1 to
	FFIBuilder
Message-ID: <20131006231024.AA3CB1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1366:51fa627e8b31
Date: 2013-10-07 01:04 +0200
http://bitbucket.org/cffi/cffi/changeset/51fa627e8b31/

Log:	Port test_distutils_package_1 to FFIBuilder

diff --git a/testing/snippets/distutils_module/setup.py b/testing/snippets/distutils_module/setup.py
--- a/testing/snippets/distutils_module/setup.py
+++ b/testing/snippets/distutils_module/setup.py
@@ -4,6 +4,7 @@
 
 import ffibuilder
 
+
 setup(
     ext_modules=[FFIExtension(ffibuilder.build_ffi)],
     cmdclass={'build_ext': build_ext},
diff --git a/testing/snippets/distutils_package_1/setup.py b/testing/snippets/distutils_package_1/setup.py
--- a/testing/snippets/distutils_package_1/setup.py
+++ b/testing/snippets/distutils_package_1/setup.py
@@ -1,7 +1,12 @@
+from distutils.core import setup
 
-from distutils.core import setup
-import snip_basic_verify1
+from cffi.packaging import FFIExtension, build_ext
+
+import snip_basic_module1.ffibuilder
+
 
 setup(
-    packages=['snip_basic_verify1'],
-    ext_modules=[snip_basic_verify1.ffi.verifier.get_extension()])
+    packages=['snip_basic_module1'],
+    ext_modules=[FFIExtension(snip_basic_module1.ffibuilder.build_ffi)],
+    cmdclass={'build_ext': build_ext},
+)
diff --git a/testing/snippets/distutils_package_1/snip_basic_module1/__init__.py b/testing/snippets/distutils_package_1/snip_basic_module1/__init__.py
new file mode 100644
diff --git a/testing/snippets/distutils_package_1/snip_basic_verify1/__init__.py b/testing/snippets/distutils_package_1/snip_basic_module1/ffibuilder.py
rename from testing/snippets/distutils_package_1/snip_basic_verify1/__init__.py
rename to testing/snippets/distutils_package_1/snip_basic_module1/ffibuilder.py
--- a/testing/snippets/distutils_package_1/snip_basic_verify1/__init__.py
+++ b/testing/snippets/distutils_package_1/snip_basic_module1/ffibuilder.py
@@ -1,17 +1,21 @@
-
-from cffi import FFI
 import sys
 
-ffi = FFI()
-ffi.cdef("""     // some declarations from the man page
+from cffi import FFIBuilder
+
+
+def build_ffi(path):
+    builder = FFIBuilder('snip_basic_module1._ffi', path)
+    builder.cdef("""     // some declarations from the man page
     struct passwd {
         char *pw_name;
         ...;
     };
     struct passwd *getpwuid(int uid);
-""")
-C = ffi.verify("""   // passed to the real C compiler
-#include <sys/types.h>
-#include <pwd.h>
-""", libraries=[],    # or a list of libraries to link with
-     force_generic_engine=hasattr(sys, '_force_generic_engine_'))
+    """)
+    builder.makelib('passwd', """   // passed to the real C compiler
+    #include <sys/types.h>
+    #include <pwd.h>
+    """, libraries=[],    # or a list of libraries to link with
+         force_generic_engine=hasattr(sys, '_force_generic_engine_'))
+    builder.write_ffi_module()
+    return builder.list_built_files()
diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py
--- a/testing/test_zintegration.py
+++ b/testing/test_zintegration.py
@@ -92,17 +92,18 @@
 
 def test_distutils_module():
     run_setup_and_program("distutils_module", '''
-    import snip_basic_module
-    lib = snip_basic_module.load_passwd()
+    import snip_basic_module as _ffi
+    lib = _ffi.load_passwd()
     p = lib.getpwuid(0)
-    assert snip_basic_module.string(p.pw_name) == b"root"
+    assert _ffi.string(p.pw_name) == b"root"
     ''')
 
 def test_distutils_package_1():
     run_setup_and_program("distutils_package_1", '''
-    import snip_basic_verify1
-    p = snip_basic_verify1.C.getpwuid(0)
-    assert snip_basic_verify1.ffi.string(p.pw_name) == b"root"
+    from snip_basic_module1 import _ffi
+    lib = _ffi.load_passwd()
+    p = lib.getpwuid(0)
+    assert _ffi.string(p.pw_name) == b"root"
     ''')
 
 def test_distutils_package_2():

From noreply at buildbot.pypy.org  Mon Oct  7 01:38:00 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Mon,  7 Oct 2013 01:38:00 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Support ext_package
Message-ID: <20131006233800.4A5631C01B0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1367:8c8fbba389fe
Date: 2013-10-07 01:36 +0200
http://bitbucket.org/cffi/cffi/changeset/8c8fbba389fe/

Log:	Support ext_package

diff --git a/cffi/packaging.py b/cffi/packaging.py
--- a/cffi/packaging.py
+++ b/cffi/packaging.py
@@ -12,12 +12,15 @@
 class build_ext(_build_ext):
     def build_extension(self, ext):
         if isinstance(ext, FFIExtension):
-            files = ext.ffi_builder(self.build_temp)
-            if not os.path.isdir(self.build_lib):
-                os.mkdir(self.build_lib)
+            pkg = self.package.split('.') if self.package else []
+            temp = os.path.join(self.build_temp, *pkg)
+            lib = os.path.join(self.build_lib, *pkg)
+
+            files = ext.ffi_builder(temp)
+            if not os.path.isdir(lib):
+                os.makedirs(lib)
             for name in files:
-                self.copy_file(
-                    os.path.join(self.build_temp, name),
-                    os.path.join(self.build_lib, name))
+                self.copy_file(os.path.join(temp, name),
+                               os.path.join(lib, name))
         else:
             super(build_ext, self).build_extension(ext)

From noreply at buildbot.pypy.org  Mon Oct  7 01:38:01 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Mon,  7 Oct 2013 01:38:01 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: We'd want to ship ffibuilder in
	the sdist
Message-ID: <20131006233801.6280A1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1368:3b261ee4ef91
Date: 2013-10-07 01:37 +0200
http://bitbucket.org/cffi/cffi/changeset/3b261ee4ef91/

Log:	We'd want to ship ffibuilder in the sdist

diff --git a/testing/snippets/distutils_module/setup.py b/testing/snippets/distutils_module/setup.py
--- a/testing/snippets/distutils_module/setup.py
+++ b/testing/snippets/distutils_module/setup.py
@@ -6,6 +6,7 @@
 
 
 setup(
+    data_files=['ffibuilder.py'],
     ext_modules=[FFIExtension(ffibuilder.build_ffi)],
     cmdclass={'build_ext': build_ext},
 )

From noreply at buildbot.pypy.org  Mon Oct  7 01:38:02 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Mon,  7 Oct 2013 01:38:02 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Port distutils_package_2 to
	FFIBuilder
Message-ID: <20131006233802.69AFD1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1369:1e3bfa8c08ae
Date: 2013-10-07 01:37 +0200
http://bitbucket.org/cffi/cffi/changeset/1e3bfa8c08ae/

Log:	Port distutils_package_2 to FFIBuilder

diff --git a/testing/snippets/distutils_package_2/setup.py b/testing/snippets/distutils_package_2/setup.py
--- a/testing/snippets/distutils_package_2/setup.py
+++ b/testing/snippets/distutils_package_2/setup.py
@@ -1,8 +1,13 @@
+from distutils.core import setup
 
-from distutils.core import setup
-import snip_basic_verify2
+from cffi.packaging import FFIExtension, build_ext
+
+import snip_basic_module2.ffibuilder
+
 
 setup(
-    packages=['snip_basic_verify2'],
-    ext_package='snip_basic_verify2',
-    ext_modules=[snip_basic_verify2.ffi.verifier.get_extension()])
+    packages=['snip_basic_module2'],
+    ext_package='snip_basic_module2',
+    ext_modules=[FFIExtension(snip_basic_module2.ffibuilder.build_ffi)],
+    cmdclass={'build_ext': build_ext},
+)
diff --git a/testing/snippets/distutils_package_2/snip_basic_module2/__init__.py b/testing/snippets/distutils_package_2/snip_basic_module2/__init__.py
new file mode 100644
diff --git a/testing/snippets/distutils_package_2/snip_basic_verify2/__init__.py b/testing/snippets/distutils_package_2/snip_basic_module2/ffibuilder.py
rename from testing/snippets/distutils_package_2/snip_basic_verify2/__init__.py
rename to testing/snippets/distutils_package_2/snip_basic_module2/ffibuilder.py
--- a/testing/snippets/distutils_package_2/snip_basic_verify2/__init__.py
+++ b/testing/snippets/distutils_package_2/snip_basic_module2/ffibuilder.py
@@ -1,18 +1,21 @@
-
-from cffi import FFI
 import sys
 
-ffi = FFI()
-ffi.cdef("""     // some declarations from the man page
+from cffi import FFIBuilder
+
+
+def build_ffi(path):
+    builder = FFIBuilder('_ffi', path)
+    builder.cdef("""     // some declarations from the man page
     struct passwd {
         char *pw_name;
         ...;
     };
     struct passwd *getpwuid(int uid);
-""")
-C = ffi.verify("""   // passed to the real C compiler
-#include <sys/types.h>
-#include <pwd.h>
-""", libraries=[],    # or a list of libraries to link with
-     ext_package='snip_basic_verify2',
-     force_generic_engine=hasattr(sys, '_force_generic_engine_'))
+    """)
+    builder.makelib('passwd', """   // passed to the real C compiler
+    #include <sys/types.h>
+    #include <pwd.h>
+    """, libraries=[],    # or a list of libraries to link with
+         force_generic_engine=hasattr(sys, '_force_generic_engine_'))
+    builder.write_ffi_module()
+    return builder.list_built_files()
diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py
--- a/testing/test_zintegration.py
+++ b/testing/test_zintegration.py
@@ -108,9 +108,10 @@
 
 def test_distutils_package_2():
     run_setup_and_program("distutils_package_2", '''
-    import snip_basic_verify2
-    p = snip_basic_verify2.C.getpwuid(0)
-    assert snip_basic_verify2.ffi.string(p.pw_name) == b"root"
+    from snip_basic_module2 import _ffi
+    lib = _ffi.load_passwd()
+    p = lib.getpwuid(0)
+    assert _ffi.string(p.pw_name) == b"root"
     ''')
 
 def test_setuptools_module():

From noreply at buildbot.pypy.org  Mon Oct  7 01:51:31 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Mon,  7 Oct 2013 01:51:31 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Port setuptools tests to FFIBuilder
Message-ID: <20131006235131.A64CF1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1370:49b84f34cf9c
Date: 2013-10-07 01:51 +0200
http://bitbucket.org/cffi/cffi/changeset/49b84f34cf9c/

Log:	Port setuptools tests to FFIBuilder

diff --git a/testing/snippets/setuptools_module/ffibuilder.py b/testing/snippets/setuptools_module/ffibuilder.py
new file mode 100644
--- /dev/null
+++ b/testing/snippets/setuptools_module/ffibuilder.py
@@ -0,0 +1,21 @@
+import sys
+
+from cffi import FFIBuilder
+
+
+def build_ffi(path):
+    builder = FFIBuilder('snip_setuptools_module', path)
+    builder.cdef("""     // some declarations from the man page
+    struct passwd {
+        char *pw_name;
+        ...;
+    };
+    struct passwd *getpwuid(int uid);
+    """)
+    builder.makelib('passwd', """   // passed to the real C compiler
+    #include <sys/types.h>
+    #include <pwd.h>
+    """, libraries=[],  # or a list of libraries to link with
+         force_generic_engine=hasattr(sys, '_force_generic_engine_'))
+    builder.write_ffi_module()
+    return builder.list_built_files()
diff --git a/testing/snippets/setuptools_module/setup.py b/testing/snippets/setuptools_module/setup.py
--- a/testing/snippets/setuptools_module/setup.py
+++ b/testing/snippets/setuptools_module/setup.py
@@ -1,8 +1,12 @@
+from setuptools import setup
 
-from setuptools import setup
-import snip_setuptools_verify
+from cffi.packaging import FFIExtension, build_ext
+
+import ffibuilder
+
 
 setup(
-    zip_safe=False,
-    py_modules=['snip_setuptools_verify'],
-    ext_modules=[snip_setuptools_verify.ffi.verifier.get_extension()])
+    data_files=['ffibuilder.py'],
+    ext_modules=[FFIExtension(ffibuilder.build_ffi)],
+    cmdclass={'build_ext': build_ext},
+)
diff --git a/testing/snippets/setuptools_module/snip_setuptools_verify.py b/testing/snippets/setuptools_module/snip_setuptools_verify.py
deleted file mode 100644
--- a/testing/snippets/setuptools_module/snip_setuptools_verify.py
+++ /dev/null
@@ -1,17 +0,0 @@
-
-from cffi import FFI
-import sys
-
-ffi = FFI()
-ffi.cdef("""     // some declarations from the man page
-    struct passwd {
-        char *pw_name;
-        ...;
-    };
-    struct passwd *getpwuid(int uid);
-""")
-C = ffi.verify("""   // passed to the real C compiler
-#include <sys/types.h>
-#include <pwd.h>
-""", libraries=[],    # or a list of libraries to link with
-     force_generic_engine=hasattr(sys, '_force_generic_engine_'))
diff --git a/testing/snippets/setuptools_package_1/setup.py b/testing/snippets/setuptools_package_1/setup.py
--- a/testing/snippets/setuptools_package_1/setup.py
+++ b/testing/snippets/setuptools_package_1/setup.py
@@ -1,8 +1,11 @@
+from setuptools import setup
 
-from setuptools import setup
-import snip_setuptools_verify1
+from cffi.packaging import FFIExtension, build_ext
+
+import snip_setuptools_module1.ffibuilder
 
 setup(
-    zip_safe=False,
-    packages=['snip_setuptools_verify1'],
-    ext_modules=[snip_setuptools_verify1.ffi.verifier.get_extension()])
+    packages=['snip_setuptools_module1'],
+    ext_modules=[FFIExtension(snip_setuptools_module1.ffibuilder.build_ffi)],
+    cmdclass={'build_ext': build_ext},
+)
diff --git a/testing/snippets/setuptools_package_1/snip_setuptools_module1/__init__.py b/testing/snippets/setuptools_package_1/snip_setuptools_module1/__init__.py
new file mode 100644
diff --git a/testing/snippets/setuptools_package_1/snip_setuptools_module1/ffibuilder.py b/testing/snippets/setuptools_package_1/snip_setuptools_module1/ffibuilder.py
new file mode 100644
--- /dev/null
+++ b/testing/snippets/setuptools_package_1/snip_setuptools_module1/ffibuilder.py
@@ -0,0 +1,21 @@
+import sys
+
+from cffi import FFIBuilder
+
+
+def build_ffi(path):
+    builder = FFIBuilder('snip_setuptools_module1._ffi', path)
+    builder.cdef("""     // some declarations from the man page
+    struct passwd {
+        char *pw_name;
+        ...;
+    };
+    struct passwd *getpwuid(int uid);
+    """)
+    builder.makelib('passwd', """   // passed to the real C compiler
+    #include <sys/types.h>
+    #include <pwd.h>
+    """, libraries=[],    # or a list of libraries to link with
+         force_generic_engine=hasattr(sys, '_force_generic_engine_'))
+    builder.write_ffi_module()
+    return builder.list_built_files()
diff --git a/testing/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py b/testing/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py
deleted file mode 100644
--- a/testing/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-
-from cffi import FFI
-import sys
-
-ffi = FFI()
-ffi.cdef("""     // some declarations from the man page
-    struct passwd {
-        char *pw_name;
-        ...;
-    };
-    struct passwd *getpwuid(int uid);
-""")
-C = ffi.verify("""   // passed to the real C compiler
-#include <sys/types.h>
-#include <pwd.h>
-""", libraries=[],    # or a list of libraries to link with
-     force_generic_engine=hasattr(sys, '_force_generic_engine_'))
diff --git a/testing/snippets/setuptools_package_2/setup.py b/testing/snippets/setuptools_package_2/setup.py
--- a/testing/snippets/setuptools_package_2/setup.py
+++ b/testing/snippets/setuptools_package_2/setup.py
@@ -1,9 +1,13 @@
+from setuptools import setup
 
-from setuptools import setup
-import snip_setuptools_verify2
+from cffi.packaging import FFIExtension, build_ext
+
+import snip_setuptools_module2.ffibuilder
+
 
 setup(
-    zip_safe=False,
-    packages=['snip_setuptools_verify2'],
-    ext_package='snip_setuptools_verify2',
-    ext_modules=[snip_setuptools_verify2.ffi.verifier.get_extension()])
+    packages=['snip_setuptools_module2'],
+    ext_package='snip_setuptools_module2',
+    ext_modules=[FFIExtension(snip_setuptools_module2.ffibuilder.build_ffi)],
+    cmdclass={'build_ext': build_ext},
+)
diff --git a/testing/snippets/setuptools_package_2/snip_setuptools_module2/__init__.py b/testing/snippets/setuptools_package_2/snip_setuptools_module2/__init__.py
new file mode 100644
diff --git a/testing/snippets/setuptools_package_2/snip_setuptools_module2/ffibuilder.py b/testing/snippets/setuptools_package_2/snip_setuptools_module2/ffibuilder.py
new file mode 100644
--- /dev/null
+++ b/testing/snippets/setuptools_package_2/snip_setuptools_module2/ffibuilder.py
@@ -0,0 +1,21 @@
+import sys
+
+from cffi import FFIBuilder
+
+
+def build_ffi(path):
+    builder = FFIBuilder('_ffi', path)
+    builder.cdef("""     // some declarations from the man page
+    struct passwd {
+        char *pw_name;
+        ...;
+    };
+    struct passwd *getpwuid(int uid);
+    """)
+    builder.makelib('passwd', """   // passed to the real C compiler
+    #include <sys/types.h>
+    #include <pwd.h>
+    """, libraries=[],    # or a list of libraries to link with
+         force_generic_engine=hasattr(sys, '_force_generic_engine_'))
+    builder.write_ffi_module()
+    return builder.list_built_files()
diff --git a/testing/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py b/testing/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py
deleted file mode 100644
--- a/testing/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-
-from cffi import FFI
-import sys
-
-ffi = FFI()
-ffi.cdef("""     // some declarations from the man page
-    struct passwd {
-        char *pw_name;
-        ...;
-    };
-    struct passwd *getpwuid(int uid);
-""")
-C = ffi.verify("""   // passed to the real C compiler
-#include <sys/types.h>
-#include <pwd.h>
-""", libraries=[],    # or a list of libraries to link with
-     ext_package='snip_setuptools_verify2',
-     force_generic_engine=hasattr(sys, '_force_generic_engine_'))
diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py
--- a/testing/test_zintegration.py
+++ b/testing/test_zintegration.py
@@ -116,21 +116,24 @@
 
 def test_setuptools_module():
     run_setup_and_program("setuptools_module", '''
-    import snip_setuptools_verify
-    p = snip_setuptools_verify.C.getpwuid(0)
-    assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root"
+    import snip_setuptools_module as _ffi
+    lib = _ffi.load_passwd()
+    p = lib.getpwuid(0)
+    assert _ffi.string(p.pw_name) == b"root"
     ''')
 
 def test_setuptools_package_1():
     run_setup_and_program("setuptools_package_1", '''
-    import snip_setuptools_verify1
-    p = snip_setuptools_verify1.C.getpwuid(0)
-    assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root"
+    from snip_setuptools_module1 import _ffi
+    lib = _ffi.load_passwd()
+    p = lib.getpwuid(0)
+    assert _ffi.string(p.pw_name) == b"root"
     ''')
 
 def test_setuptools_package_2():
     run_setup_and_program("setuptools_package_2", '''
-    import snip_setuptools_verify2
-    p = snip_setuptools_verify2.C.getpwuid(0)
-    assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root"
+    from snip_setuptools_module2 import _ffi
+    lib = _ffi.load_passwd()
+    p = lib.getpwuid(0)
+    assert _ffi.string(p.pw_name) == b"root"
     ''')

From noreply at buildbot.pypy.org  Mon Oct  7 01:59:04 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Mon,  7 Oct 2013 01:59:04 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Bootstrapping documentation
Message-ID: <20131006235904.646BE1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: split-verify
Changeset: r1371:d18e79b97f2e
Date: 2013-10-07 01:58 +0200
http://bitbucket.org/cffi/cffi/changeset/d18e79b97f2e/

Log:	Bootstrapping documentation

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -355,6 +355,21 @@
 for more details about the ``verifier`` object.
 
 
+Bootstrapping CFFI
+------------------
+
+Your ``setup.py`` now needs CFFI to be importable at build time.
+You can let tell ``setuptools`` to download it before you import
+anything from CFFI::
+
+  from setuptools import setup, Distribution
+
+  Distribution(attrs=dict(setup_requires=['cffi']))
+
+  from cffi.packaging import FFIExtension, build_ext
+  ...
+
+
 Cleaning up the __pycache__ directory
 -------------------------------------
 

From noreply at buildbot.pypy.org  Mon Oct  7 09:09:06 2013
From: noreply at buildbot.pypy.org (chrish42)
Date: Mon,  7 Oct 2013 09:09:06 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-2.7.5: Raise an error when finding a C0
 control character in JSON string for pypyjson implementation. Fixes new,
 failing test in stdlib-2.7.5.
Message-ID: <20131007070906.E86BF1C00EC@cobra.cs.uni-duesseldorf.de>

Author: Christian Hudon <chrish at pianocktail.org>
Branch: stdlib-2.7.5
Changeset: r67170:4b9d6834816b
Date: 2013-09-29 23:07 -0400
http://bitbucket.org/pypy/pypy/changeset/4b9d6834816b/

Log:	Raise an error when finding a C0 control character in JSON string
	for pypyjson implementation. Fixes new, failing test in
	stdlib-2.7.5.

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -317,6 +317,8 @@
                 return self.decode_string_escaped(start, content_so_far)
             elif ch == '\0':
                 self._raise("Unterminated string starting at char %d", start)
+            elif ch < '\x20' or ch == '\x7f':
+                self._raise("Invalid control character at char %d", self.pos-1)
 
 
     def decode_string_escaped(self, start, content_so_far):
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -185,4 +185,8 @@
         res = _pypyjson.loads('"z\\ud834\\udd20x"')
         assert res == expected
 
-
+    def test_tab_in_string_should_fail(self):
+        import _pypyjson
+        # http://json.org/JSON_checker/test/fail25.json
+        s = '["\ttab\tcharacter\tin\tstring\t"]'
+        raises(ValueError, "_pypyjson.loads(s)")

From noreply at buildbot.pypy.org  Mon Oct  7 09:09:08 2013
From: noreply at buildbot.pypy.org (chrish42)
Date: Mon,  7 Oct 2013 09:09:08 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-2.7.5: Remove redundant test for NULL;
 emulate behavior of CPython 2.7 by only raising an error for
 control characters < 0x20.
Message-ID: <20131007070908.2FB5C1C00EC@cobra.cs.uni-duesseldorf.de>

Author: Christian Hudon <chrish at pianocktail.org>
Branch: stdlib-2.7.5
Changeset: r67171:1307a5424887
Date: 2013-10-06 17:31 -0400
http://bitbucket.org/pypy/pypy/changeset/1307a5424887/

Log:	Remove redundant test for NULL; emulate behavior of CPython 2.7 by
	only raising an error for control characters < 0x20.

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -315,9 +315,7 @@
                 content_so_far = self.getslice(start, i-1)
                 self.pos = i-1
                 return self.decode_string_escaped(start, content_so_far)
-            elif ch == '\0':
-                self._raise("Unterminated string starting at char %d", start)
-            elif ch < '\x20' or ch == '\x7f':
+            elif ch < '\x20':
                 self._raise("Invalid control character at char %d", self.pos-1)
 
 

From noreply at buildbot.pypy.org  Mon Oct  7 10:30:13 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon,  7 Oct 2013 10:30:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: factor out the result computation into
 its own function. This can almost go
Message-ID: <20131007083013.255491C014D@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r67172:bd04f5c5c30e
Date: 2013-10-04 17:13 +0200
http://bitbucket.org/pypy/pypy/changeset/bd04f5c5c30e/

Log:	factor out the result computation into its own function. This can
	almost go into rlib, apart from error handling.

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -424,21 +424,24 @@
     x = w_float1.floatval
     y = w_float2.floatval
 
+    return W_FloatObject(_pow(space, x, y))
+
+def _pow(space, x, y):
     # Sort out special cases here instead of relying on pow()
-    if y == 2.0:                      # special case for performance:
-        return W_FloatObject(x * x)   # x * x is always correct
+    if y == 2.0:       # special case for performance:
+        return x * x   # x * x is always correct
     if y == 0.0:
         # x**0 is 1, even 0**0
-        return W_FloatObject(1.0)
+        return 1.0
     if isnan(x):
         # nan**y = nan, unless y == 0
-        return W_FloatObject(x)
+        return x
     if isnan(y):
         # x**nan = nan, unless x == 1; x**nan = x
         if x == 1.0:
-            return W_FloatObject(1.0)
+            return 1.0
         else:
-            return W_FloatObject(y)
+            return y
     if isinf(y):
         # x**inf is: 0.0 if abs(x) < 1; 1.0 if abs(x) == 1; inf if
         # abs(x) > 1 (including case where x infinite)
@@ -447,11 +450,11 @@
         # abs(x) > 1 (including case where v infinite)
         x = abs(x)
         if x == 1.0:
-            return W_FloatObject(1.0)
+            return 1.0
         elif (y > 0.0) == (x > 1.0):
-            return W_FloatObject(INFINITY)
+            return INFINITY
         else:
-            return W_FloatObject(0.0)
+            return 0.0
     if isinf(x):
         # (+-inf)**w is: inf for w positive, 0 for w negative; in oth
         # cases, we need to add the appropriate sign if w is an odd
@@ -459,14 +462,14 @@
         y_is_odd = math.fmod(abs(y), 2.0) == 1.0
         if y > 0.0:
             if y_is_odd:
-                return W_FloatObject(x)
+                return x
             else:
-                return W_FloatObject(abs(x))
+                return abs(x)
         else:
             if y_is_odd:
-                return W_FloatObject(copysign(0.0, x))
+                return copysign(0.0, x)
             else:
-                return W_FloatObject(0.0)
+                return 0.0
 
     if x == 0.0:
         if y < 0.0:
@@ -480,7 +483,7 @@
     # -           pipermail/python-bugs-list/2003-March/016795.html
     if x < 0.0:
         if isnan(y):
-            return W_FloatObject(NAN)
+            return NAN
         if math.floor(y) != y:
             raise OperationError(space.w_ValueError,
                                  space.wrap("negative number cannot be "
@@ -494,9 +497,9 @@
     if x == 1.0:
         # (-1) ** large_integer also ends up here
         if negate_result:
-            return W_FloatObject(-1.0)
+            return -1.0
         else:
-            return W_FloatObject(1.0)
+            return 1.0
 
     try:
         # We delegate to our implementation of math.pow() the error detection.
@@ -510,7 +513,7 @@
 
     if negate_result:
         z = -z
-    return W_FloatObject(z)
+    return z
 
 
 def neg__Float(space, w_float1):

From noreply at buildbot.pypy.org  Mon Oct  7 14:00:53 2013
From: noreply at buildbot.pypy.org (jerith)
Date: Mon,  7 Oct 2013 14:00:53 +0200 (CEST)
Subject: [pypy-commit] cffi split-verify: Experimental serialisation of
 cffi.model types into Python code that instantiates them.
Message-ID: <20131007120053.044FB1C0209@cobra.cs.uni-duesseldorf.de>

Author: Jeremy Thurgood <firxen at gmail.com>
Branch: split-verify
Changeset: r1372:ce63a7e3e315
Date: 2013-10-07 13:46 +0200
http://bitbucket.org/cffi/cffi/changeset/ce63a7e3e315/

Log:	Experimental serialisation of cffi.model types into Python code that
	instantiates them.

diff --git a/cffi/builder.py b/cffi/builder.py
--- a/cffi/builder.py
+++ b/cffi/builder.py
@@ -1,7 +1,7 @@
 import os
-import pickle
 
 from .api import FFI
+from . import model
 
 
 MODULE_BOILERPLATE = """
@@ -10,7 +10,7 @@
 #####                                                                  #####
 
 import pickle
-from cffi import FFI
+from cffi import FFI, model
 
 
 _ffi = FFI()
@@ -67,6 +67,60 @@
 """
 
 
+class NotReadyYet(Exception):
+    pass
+
+
+class DeclarationBuilder(object):
+    def __init__(self, model_object, built_declarations, our_declarations):
+        self._model_object = model_object
+        self._built_declarations = built_declarations
+        self._our_declarations = our_declarations
+
+    def _format_param(self, param):
+        if isinstance(param, model.BaseTypeByIdentity):
+            od = (type(param), getattr(param, 'name', None))
+            if od not in self._our_declarations:
+                return DeclarationBuilder(
+                    param, self._built_declarations,
+                    self._our_declarations).build()
+            if param not in self._built_declarations:
+                raise NotReadyYet()
+            return "declarations[%r]" % self._built_declarations[param]
+        if isinstance(param, tuple):
+            return '(%s,)' % ', '.join(self._format_param(p) for p in param)
+        return repr(param)
+
+    def build(self):
+        try:
+            params = [(k, self._format_param(v))
+                      for k, v in self._model_object._get_items()]
+            if isinstance(self._model_object, model.StructOrUnion):
+                params.extend([
+                    ('fldnames', self._format_param(
+                        self._model_object.fldnames)),
+                    ('fldtypes', self._format_param(
+                        self._model_object.fldtypes)),
+                    ('fldbitsize', self._format_param(
+                        self._model_object.fldbitsize)),
+                ])
+            elif isinstance(self._model_object, model.EnumType):
+                params.extend([
+                    ('enumerators', self._format_param(
+                        self._model_object.enumerators)),
+                    ('enumvalues', self._format_param(
+                        self._model_object.enumvalues)),
+                    ('baseinttype', self._format_param(
+                        self._model_object.baseinttype)),
+                ])
+        except NotReadyYet:
+            return None
+
+        return "model.%s(%s)" % (
+            self._model_object.__class__.__name__, ', '.join(
+                '%s=%s' % param for param in params))
+
+
 class FFIBuilder(object):
     def __init__(self, module_name, build_path, backend=None):
         module_package = ''
@@ -104,11 +158,47 @@
         self.ffi.verifier.make_library(libfile_build_path)
         self._module_source += MAKELIB_FUNC_TEMPLATE % (libname, barefilename)
         self._built_files.append(libfile_path)
+        return self.ffi.verifier._load_library()
+
+    def _write_declarations(self):
+        self._module_source += "def _make_declarations():\n"
+        self._module_source += "    declarations = {}\n"
+
+        declarations = self.ffi._parser._declarations
+        our_decls = set((type(obj), getattr(obj, 'name', None))
+                        for obj in declarations.values())
+        built_decls = {}
+        decls = [(k, DeclarationBuilder(v, built_decls, our_decls))
+                 for k, v in self.ffi._parser._declarations.items()]
+
+        max_tries = (len(decls) + 1) ** 2 / 2
+
+        tries = 0
+        while decls:
+            tries += 1
+            if tries > max_tries:
+                raise Exception("Problem serialising declarations.")
+            name, dbuilder = decls.pop(0)
+            instantiation = dbuilder.build()
+            if instantiation is None:
+                decls.append((name, dbuilder))
+            else:
+                built_decls[dbuilder._model_object] = name
+                self._module_source += "    declarations[%r] = %s\n" % (
+                    name, instantiation)
+                if getattr(dbuilder._model_object, 'partial_resolved', None):
+                    self._module_source += (
+                        "    declarations[%r].partial = True\n" % (name,))
+                    self._module_source += (
+                        "    declarations[%r].partial_resolved = True\n" % (
+                            name,))
+
+        self._module_source += "    return declarations\n\n"
+        self._module_source += (
+            "_ffi._parser._declarations = _make_declarations()\n")
 
     def write_ffi_module(self):
-        self._module_source += (
-            "_ffi._parser._declarations = pickle.loads(%r)\n" %
-            pickle.dumps(self.ffi._parser._declarations, 2))
+        self._write_declarations()
         try:
             os.makedirs(self._build_path)
         except OSError:
diff --git a/testing/test_makelib.py b/testing/test_makelib.py
--- a/testing/test_makelib.py
+++ b/testing/test_makelib.py
@@ -119,10 +119,12 @@
 def test_ffi_do_some_stuff(tmpdir):
     builder = FFIBuilder("foo_ffi", str(tmpdir))
     builder.cdef("""
+        enum ee { EE1, EE2, EE3, ... };
         struct foo_s { int x; int y; };
         int grid_distance(struct foo_s offset);
     """)
     builder.makelib('foo', """
+        enum ee { EE1=10, EE2, EE3=-10, EE4 };
         struct foo_s { int x; int y; };
         int grid_distance(struct foo_s offset) {
             return offset.x + offset.y;
@@ -142,6 +144,8 @@
     assert foo_ffi.alignof('struct foo_s') == foo_ffi.sizeof('int')
     assert foo_ffi.typeof(foo_ffi.cast('long', 42)) == foo_ffi.typeof('long')
     assert foo_ffi.string(foo_ffi.new('char *', b"\x00")) == b""
+    assert foo_ffi.string(foo_ffi.cast('enum ee', 11)) == "EE2"
+    assert foo_ffi.string(foo_ffi.cast('enum ee', -10)) == "EE3"
 
     def cb(n):
         return n + 1

From noreply at buildbot.pypy.org  Mon Oct  7 17:17:51 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  7 Oct 2013 17:17:51 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: flush
Message-ID: <20131007151751.E9CF21C135F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67181:35758f162380
Date: 2013-10-07 17:10 +0200
http://bitbucket.org/pypy/pypy/changeset/35758f162380/

Log:	flush

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -156,3 +156,16 @@
                 raise OSError(errno, os.strerror(errno))
             return res
         raise ValueError("I/O operation on closed file")
+
+    def flush(self):
+        if self.ll_file:
+            res = c_fflush(self.ll_file)
+            if res != 0:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return
+        raise ValueError("I/O operation on closed file")
+
+    def __del__(self):
+        self.close()
+
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -106,3 +106,16 @@
         res = self.interpret(f, [])
         assert res == 3
 
+    def test_flush(self):
+        fname = str(self.tmpdir.join('file_trunc'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            f.flush()
+            f2 = open(fname)
+            assert f2.read() == "xyz"
+            f2.close()
+            f.close()
+
+        self.interpret(f, [])
diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -11,10 +11,10 @@
 
 import py
 # clean up early rpython/_cache
-try:
-    py.path.local(cache_dir).remove()
-except Exception:
-    pass
+#try:
+#    py.path.local(cache_dir).remove()
+#except Exception:
+#    pass
 
 from rpython.config.config import (to_optparse, OptionDescription, BoolOption,
     ArbitraryOption, StrOption, IntOption, Config, ChoiceOption, OptHelpFormatter)

From noreply at buildbot.pypy.org  Mon Oct  7 17:17:53 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  7 Oct 2013 17:17:53 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: ftruncate
Message-ID: <20131007151753.36C771C1380@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67182:d55b0202cef0
Date: 2013-10-07 17:16 +0200
http://bitbucket.org/pypy/pypy/changeset/d55b0202cef0/

Log:	ftruncate

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -6,18 +6,27 @@
 
 import os
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.tool import rffi_platform as platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rlib import rposix
 from rpython.rlib.rstring import StringBuilder
 
-eci = ExternalCompilationInfo(includes=['stdio.h'])
+eci = ExternalCompilationInfo(includes=['stdio.h', 'unistd.h', 'sys/types.h'])
 
 def llexternal(*args):
     return rffi.llexternal(*args, compilation_info=eci)
 
 FILE = lltype.Struct('FILE') # opaque type maybe
 
+class CConfig(object):
+    _compilation_info_ = eci
+
+    off_t = platform.SimpleType('off_t')
+
+CC = platform.configure(CConfig)
+OFF_T = CC['off_t']
+
 c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
 c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
 c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
@@ -32,7 +41,8 @@
 c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
 c_fileno = llexternal('fileno', [lltype.Ptr(FILE)], rffi.INT)
 c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], lltype.Signed)
-c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], lltype.Signed)
+c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT)
+c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT)
 
 BASE_BUF_SIZE = 4096
 
@@ -166,6 +176,17 @@
             return
         raise ValueError("I/O operation on closed file")
 
+    def truncate(self, arg=-1):
+        if self.ll_file:
+            if arg == -1:
+                arg = self.tell()
+            res = c_ftruncate(self.fileno(), arg)
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return
+        raise ValueError("I/O operation on closed file")
+
     def __del__(self):
         self.close()
 
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -107,7 +107,7 @@
         assert res == 3
 
     def test_flush(self):
-        fname = str(self.tmpdir.join('file_trunc'))
+        fname = str(self.tmpdir.join('file_flush'))
 
         def f():
             f = open(fname, "w")
@@ -119,3 +119,19 @@
             f.close()
 
         self.interpret(f, [])
+
+    def test_truncate(self):
+        fname = str(self.tmpdir.join('file_trunc'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            f.seek(0)
+            f.truncate(2)
+            f.close()
+            f2 = open(fname)
+            assert f2.read() == "xy"
+            f2.close()
+
+        f()
+        self.interpret(f, [])

From noreply at buildbot.pypy.org  Mon Oct  7 17:18:52 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  7 Oct 2013 17:18:52 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: merge default
Message-ID: <20131007151852.81DC11C135F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67183:44224be36e53
Date: 2013-10-07 17:18 +0200
http://bitbucket.org/pypy/pypy/changeset/44224be36e53/

Log:	merge default

diff --git a/pypy/TODO b/pypy/TODO
deleted file mode 100644
--- a/pypy/TODO
+++ /dev/null
@@ -1,2 +0,0 @@
-
-* ARM
diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -149,14 +149,17 @@
          only used if the array is constructed that way. Almost always this
          parameter is NULL.
     """
-    if min_depth !=0 or max_depth != 0:
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            '_PyArray_FromAny called with not-implemented min_dpeth or max_depth argument'))
     if requirements not in (0, NPY_DEFAULT):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             '_PyArray_FromAny called with not-implemented requirements argument'))
     w_array = array(space, w_obj, w_dtype=w_dtype, copy=False)
-    if w_array.is_scalar():
+    if min_depth !=0 and len(w_array.get_shape()) < min_depth:
+        raise OperationError(space.w_ValueError, space.wrap(
+            'object of too small depth for desired array'))
+    elif max_depth !=0 and len(w_array.get_shape()) > max_depth:
+        raise OperationError(space.w_ValueError, space.wrap(
+            'object of too deep for desired array'))
+    elif w_array.is_scalar():
         # since PyArray_DATA() fails on scalars, create a 1D array and set empty
         # shape. So the following combination works for *reading* scalars:
         #     PyObject *arr = PyArray_FromAny(obj);
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -90,15 +90,16 @@
     def test_FromAny(self, space, api):
         a = array(space, [10, 5, 3])
         assert api._PyArray_FromAny(a, NULL, 0, 0, 0, NULL) is a
-        self.raises(space, api, NotImplementedError, api._PyArray_FromAny,
-                    a, NULL, 0, 3, 0, NULL)
+        assert api._PyArray_FromAny(a, NULL, 1, 4, 0, NULL) is a
+        self.raises(space, api, ValueError, api._PyArray_FromAny,
+                    a, NULL, 4, 5, 0, NULL)
 
     def test_FromObject(self, space, api):
         a = array(space, [10, 5, 3])
         assert api._PyArray_FromObject(a, a.get_dtype().num, 0, 0) is a
-        exc = self.raises(space, api, NotImplementedError, api._PyArray_FromObject,
-                    a, 11, 0, 3)
-        assert exc.errorstr(space).find('FromObject') >= 0
+        exc = self.raises(space, api, ValueError, api._PyArray_FromObject,
+                    a, 11, 4, 5)
+        assert exc.errorstr(space).find('desired') >= 0
 
     def test_list_from_fixedptr(self, space, api):
         A = lltype.GcArray(lltype.Float)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -277,3 +277,28 @@
             f1 = call_release_gil(..., descr=<Calli 4 ii EF=6 OS=62>)
             ...
         """)
+
+    def test__cffi_bug1(self):
+        from rpython.rlib.test.test_clibffi import get_libm_name
+        def main(libm_name):
+            try:
+                import _cffi_backend
+            except ImportError:
+                sys.stderr.write('SKIP: cannot import _cffi_backend\n')
+                return 0
+
+            libm = _cffi_backend.load_library(libm_name)
+            BDouble = _cffi_backend.new_primitive_type("double")
+            BSin = _cffi_backend.new_function_type([BDouble], BDouble)
+            sin = libm.load_function(BSin, 'sin')
+
+            def f(*args):
+                for i in range(300):
+                    sin(*args)
+
+            f(1.0)
+            f(1)
+        #
+        libm_name = get_libm_name(sys.platform)
+        log = self.run(main, [libm_name])
+        # assert did not crash
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -424,21 +424,24 @@
     x = w_float1.floatval
     y = w_float2.floatval
 
+    return W_FloatObject(_pow(space, x, y))
+
+def _pow(space, x, y):
     # Sort out special cases here instead of relying on pow()
-    if y == 2.0:                      # special case for performance:
-        return W_FloatObject(x * x)   # x * x is always correct
+    if y == 2.0:       # special case for performance:
+        return x * x   # x * x is always correct
     if y == 0.0:
         # x**0 is 1, even 0**0
-        return W_FloatObject(1.0)
+        return 1.0
     if isnan(x):
         # nan**y = nan, unless y == 0
-        return W_FloatObject(x)
+        return x
     if isnan(y):
         # x**nan = nan, unless x == 1; x**nan = x
         if x == 1.0:
-            return W_FloatObject(1.0)
+            return 1.0
         else:
-            return W_FloatObject(y)
+            return y
     if isinf(y):
         # x**inf is: 0.0 if abs(x) < 1; 1.0 if abs(x) == 1; inf if
         # abs(x) > 1 (including case where x infinite)
@@ -447,11 +450,11 @@
         # abs(x) > 1 (including case where v infinite)
         x = abs(x)
         if x == 1.0:
-            return W_FloatObject(1.0)
+            return 1.0
         elif (y > 0.0) == (x > 1.0):
-            return W_FloatObject(INFINITY)
+            return INFINITY
         else:
-            return W_FloatObject(0.0)
+            return 0.0
     if isinf(x):
         # (+-inf)**w is: inf for w positive, 0 for w negative; in oth
         # cases, we need to add the appropriate sign if w is an odd
@@ -459,14 +462,14 @@
         y_is_odd = math.fmod(abs(y), 2.0) == 1.0
         if y > 0.0:
             if y_is_odd:
-                return W_FloatObject(x)
+                return x
             else:
-                return W_FloatObject(abs(x))
+                return abs(x)
         else:
             if y_is_odd:
-                return W_FloatObject(copysign(0.0, x))
+                return copysign(0.0, x)
             else:
-                return W_FloatObject(0.0)
+                return 0.0
 
     if x == 0.0:
         if y < 0.0:
@@ -480,7 +483,7 @@
     # -           pipermail/python-bugs-list/2003-March/016795.html
     if x < 0.0:
         if isnan(y):
-            return W_FloatObject(NAN)
+            return NAN
         if math.floor(y) != y:
             raise OperationError(space.w_ValueError,
                                  space.wrap("negative number cannot be "
@@ -494,9 +497,9 @@
     if x == 1.0:
         # (-1) ** large_integer also ends up here
         if negate_result:
-            return W_FloatObject(-1.0)
+            return -1.0
         else:
-            return W_FloatObject(1.0)
+            return 1.0
 
     try:
         # We delegate to our implementation of math.pow() the error detection.
@@ -510,7 +513,7 @@
 
     if negate_result:
         z = -z
-    return W_FloatObject(z)
+    return z
 
 
 def neg__Float(space, w_float1):
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -381,6 +381,8 @@
             res = self.llinterp.eval_graph(ptr._obj.graph, args)
         else:
             res = ptr._obj._callable(*args)
+        if RESULT is lltype.Void:
+            return None
         return support.cast_result(RESULT, res)
 
     def _do_call(self, func, args_i, args_r, args_f, calldescr):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -45,6 +45,15 @@
             return value
         return OptValue(self.force_box(optforce))
 
+    def get_args_for_fail(self, modifier):
+        # checks for recursion: it is False unless
+        # we have already seen the very same keybox
+        if self.box is None and not modifier.already_seen_virtual(self.keybox):
+            self._get_args_for_fail(modifier)
+
+    def _get_args_for_fail(self, modifier):
+        raise NotImplementedError("abstract base")
+
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
             return self._make_virtual(modifier)
@@ -193,16 +202,13 @@
             self._cached_sorted_fields = lst
         return lst
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            # checks for recursion: it is False unless
-            # we have already seen the very same keybox
-            lst = self._get_field_descr_list()
-            fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst]
-            modifier.register_virtual_fields(self.keybox, fieldboxes)
-            for ofs in lst:
-                fieldvalue = self._fields[ofs]
-                fieldvalue.get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        lst = self._get_field_descr_list()
+        fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst]
+        modifier.register_virtual_fields(self.keybox, fieldboxes)
+        for ofs in lst:
+            fieldvalue = self._fields[ofs]
+            fieldvalue.get_args_for_fail(modifier)
 
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
@@ -254,18 +260,15 @@
     def set_item_value(self, i, newval):
         raise NotImplementedError
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            # checks for recursion: it is False unless
-            # we have already seen the very same keybox
-            itemboxes = []
-            for i in range(self.getlength()):
-                itemvalue = self.get_item_value(i)
-                itemboxes.append(itemvalue.get_key_box())
-            modifier.register_virtual_fields(self.keybox, itemboxes)
-            for i in range(self.getlength()):
-                itemvalue = self.get_item_value(i)
-                itemvalue.get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        itemboxes = []
+        for i in range(self.getlength()):
+            itemvalue = self.get_item_value(i)
+            itemboxes.append(itemvalue.get_key_box())
+        modifier.register_virtual_fields(self.keybox, itemboxes)
+        for i in range(self.getlength()):
+            itemvalue = self.get_item_value(i)
+            itemvalue.get_args_for_fail(modifier)
 
 
 class VArrayValue(AbstractVArrayValue):
@@ -370,17 +373,16 @@
             descrs.append(item_descrs)
         return descrs
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            itemdescrs = self._get_list_of_descrs()
-            itemboxes = []
-            for i in range(len(self._items)):
-                for descr in itemdescrs[i]:
-                    itemboxes.append(self._items[i][descr].get_key_box())
-            modifier.register_virtual_fields(self.keybox, itemboxes)
-            for i in range(len(self._items)):
-                for descr in itemdescrs[i]:
-                    self._items[i][descr].get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        itemdescrs = self._get_list_of_descrs()
+        itemboxes = []
+        for i in range(len(self._items)):
+            for descr in itemdescrs[i]:
+                itemboxes.append(self._items[i][descr].get_key_box())
+        modifier.register_virtual_fields(self.keybox, itemboxes)
+        for i in range(len(self._items)):
+            for descr in itemdescrs[i]:
+                self._items[i][descr].get_args_for_fail(modifier)
 
     def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
@@ -481,6 +483,15 @@
     def getitem_raw(self, offset, length, descr):
         return self.rawbuffer_value.getitem_raw(self.offset+offset, length, descr)
 
+    def _get_args_for_fail(self, modifier):
+        box = self.rawbuffer_value.get_key_box()
+        modifier.register_virtual_fields(self.keybox, [box])
+        self.rawbuffer_value.get_args_for_fail(modifier)
+
+    def _make_virtual(self, modifier):
+        return modifier.make_vrawslice(self.offset)
+
+
 class OptVirtualize(optimizer.Optimization):
     "Virtualize objects until they escape."
 
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -284,7 +284,10 @@
         return VArrayStructInfo(arraydescr, fielddescrs)
 
     def make_vrawbuffer(self, size, offsets, descrs):
-        return VRawBufferStateInfo(size, offsets, descrs)
+        return VRawBufferInfo(size, offsets, descrs)
+
+    def make_vrawslice(self, offset):
+        return VRawSliceInfo(offset)
 
     def make_vstrplain(self, is_unicode=False):
         if is_unicode:
@@ -554,10 +557,13 @@
             debug_print("\t\t", str(untag(i)))
 
 
-class VRawBufferStateInfo(AbstractVirtualInfo):
+class VAbstractRawInfo(AbstractVirtualInfo):
     kind = INT
     is_about_raw = True
 
+
+class VRawBufferInfo(VAbstractRawInfo):
+
     def __init__(self, size, offsets, descrs):
         self.size = size
         self.offsets = offsets
@@ -580,6 +586,25 @@
             debug_print("\t\t", str(untag(i)))
 
 
+class VRawSliceInfo(VAbstractRawInfo):
+
+    def __init__(self, offset):
+        self.offset = offset
+
+    @specialize.argtype(1)
+    def allocate_int(self, decoder, index):
+        assert len(self.fieldnums) == 1
+        base_buffer = decoder.decode_int(self.fieldnums[0])
+        buffer = decoder.int_add_const(base_buffer, self.offset)
+        decoder.virtuals_cache.set_int(index, buffer)
+        return buffer
+
+    def debug_prints(self):
+        debug_print("\tvrawsliceinfo", " at ",  compute_unique_id(self))
+        for i in self.fieldnums:
+            debug_print("\t\t", str(untag(i)))
+
+
 class VArrayStructInfo(AbstractVirtualInfo):
     def __init__(self, arraydescr, fielddescrs):
         self.arraydescr = arraydescr
@@ -783,7 +808,8 @@
         v = self.virtuals_cache.get_int(index)
         if not v:
             v = self.rd_virtuals[index]
-            assert v.is_about_raw and isinstance(v, VRawBufferStateInfo)
+            ll_assert(bool(v), "resume.py: null rd_virtuals[index]")
+            assert v.is_about_raw and isinstance(v, VAbstractRawInfo)
             v = v.allocate_int(self, index)
             ll_assert(v == self.virtuals_cache.get_int(index), "resume.py: bad cache")
         return v
@@ -1116,6 +1142,10 @@
     def write_a_float(self, index, box):
         self.boxes_f[index] = box
 
+    def int_add_const(self, intbox, offset):
+        return self.metainterp.execute_and_record(rop.INT_ADD, None, intbox,
+                                                  ConstInt(offset))
+
 # ---------- when resuming for blackholing, get direct values ----------
 
 def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage,
@@ -1407,6 +1437,9 @@
     def write_a_float(self, index, float):
         self.blackholeinterp.setarg_f(index, float)
 
+    def int_add_const(self, base, offset):
+        return base + offset
+
 # ____________________________________________________________
 
 def dump_storage(storage, liveboxes):
diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py
--- a/rpython/jit/metainterp/test/test_fficall.py
+++ b/rpython/jit/metainterp/test/test_fficall.py
@@ -86,15 +86,17 @@
             data = rffi.ptradd(exchange_buffer, ofs)
             rffi.cast(lltype.Ptr(TYPE), data)[0] = write_rvalue
 
-        def f():
+        def f(i):
             exbuf = lltype.malloc(rffi.CCHARP.TO, (len(avalues)+2) * 16,
-                                  flavor='raw', zero=True)
-            ofs = 16
+                                  flavor='raw')
+
+            targetptr = rffi.ptradd(exbuf, 16)
             for avalue in unroll_avalues:
                 TYPE = rffi.CArray(lltype.typeOf(avalue))
-                data = rffi.ptradd(exbuf, ofs)
-                rffi.cast(lltype.Ptr(TYPE), data)[0] = avalue
-                ofs += 16
+                if i >= 9:    # a guard that can fail
+                    pass
+                rffi.cast(lltype.Ptr(TYPE), targetptr)[0] = avalue
+                targetptr = rffi.ptradd(targetptr, 16)
 
             jit_ffi_call(cif_description, func_addr, exbuf)
 
@@ -102,8 +104,7 @@
                 res = 654321
             else:
                 TYPE = rffi.CArray(lltype.typeOf(rvalue))
-                data = rffi.ptradd(exbuf, ofs)
-                res = rffi.cast(lltype.Ptr(TYPE), data)[0]
+                res = rffi.cast(lltype.Ptr(TYPE), targetptr)[0]
             lltype.free(exbuf, flavor='raw')
             if lltype.typeOf(res) is lltype.SingleFloat:
                 res = float(res)
@@ -117,9 +118,9 @@
             return res == rvalue
 
         with FakeFFI(fake_call_impl_any):
-            res = f()
+            res = f(-42)
             assert matching_result(res, rvalue)
-            res = self.interp_operations(f, [],
+            res = self.interp_operations(f, [-42],
                             supports_floats = supports_floats,
                           supports_longlong = supports_longlong,
                       supports_singlefloats = supports_singlefloats)
@@ -132,6 +133,19 @@
             self.check_operations_history(call_may_force=0,
                                           call_release_gil=expected_call_release_gil)
 
+            ##################################################
+            driver = jit.JitDriver(reds=['i'], greens=[])
+            def main():
+                i = 0
+                while 1:
+                    driver.jit_merge_point(i=i)
+                    res = f(i)
+                    i += 1
+                    if i == 12:
+                        return res
+            self.meta_interp(main, [])
+
+
     def test_simple_call_int(self):
         self._run([types.signed] * 2, types.signed, [456, 789], -42)
 
diff --git a/rpython/memory/support.py b/rpython/memory/support.py
--- a/rpython/memory/support.py
+++ b/rpython/memory/support.py
@@ -121,13 +121,15 @@
                 cur = next
             free_non_gc_object(self)
 
-        def _length_estimate(self):
+        def length(self):
             chunk = self.chunk
+            result = 0
             count = self.used_in_last_chunk
             while chunk:
+                result += count
                 chunk = chunk.next
-                count += chunk_size
-            return count
+                count = chunk_size
+            return result
 
         def foreach(self, callback, arg):
             """Invoke 'callback(address, arg)' for all addresses in the stack.
@@ -144,7 +146,7 @@
         foreach._annspecialcase_ = 'specialize:arg(1)'
 
         def stack2dict(self):
-            result = AddressDict(self._length_estimate())
+            result = AddressDict(self.length())
             self.foreach(_add_in_dict, result)
             return result
 
diff --git a/rpython/memory/test/test_support.py b/rpython/memory/test/test_support.py
--- a/rpython/memory/test/test_support.py
+++ b/rpython/memory/test/test_support.py
@@ -94,6 +94,18 @@
             assert a == addrs[i]
         assert not ll.non_empty()
 
+    def test_length(self):
+        AddressStack = get_address_stack(10)
+        ll = AddressStack()
+        a = raw_malloc(llmemory.sizeof(lltype.Signed))
+        for i in range(42):
+            assert ll.length() == i
+            ll.append(a)
+        for i in range(42-1, -1, -1):
+            b = ll.pop()
+            assert b == a
+            assert ll.length() == i
+
 
 class TestAddressDeque:
     def test_big_access(self):
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -731,10 +731,16 @@
             if c.numdigits() == 1 and c._digits[0] == ONEDIGIT:
                 return NULLRBIGINT
 
-            # if base < 0:
-            #     base = base % modulus
-            # Having the base positive just makes things easier.
-            if a.sign < 0:
+            # Reduce base by modulus in some cases:
+            # 1. If base < 0.  Forcing the base non-neg makes things easier.
+            # 2. If base is obviously larger than the modulus.  The "small
+            #    exponent" case later can multiply directly by base repeatedly,
+            #    while the "large exponent" case multiplies directly by base 31
+            #    times.  It can be unboundedly faster to multiply by
+            #    base % modulus instead.
+            # We could _always_ do this reduction, but mod() isn't cheap,
+            # so we only do it when it buys something.
+            if a.sign < 0 or a.numdigits() > c.numdigits():
                 a = a.mod(c)
 
         elif b.sign == 0:
diff --git a/rpython/rlib/rpath.py b/rpython/rlib/rpath.py
--- a/rpython/rlib/rpath.py
+++ b/rpython/rlib/rpath.py
@@ -23,3 +23,41 @@
             return path
 else:
     raise ImportError('Unsupported os: %s' % os.name)
+
+
+def dirname(p):
+    """Returns the directory component of a pathname"""
+    i = p.rfind('/') + 1
+    assert i >= 0
+    head = p[:i]
+    if head and head != '/' * len(head):
+        head = head.rstrip('/')
+    return head
+
+
+def basename(p):
+    """Returns the final component of a pathname"""
+    i = p.rfind('/') + 1
+    assert i >= 0
+    return p[i:]
+
+
+def split(p):
+    """Split a pathname.  Returns tuple "(head, tail)" where "tail" is
+    everything after the final slash.  Either part may be empty."""
+    i = p.rfind('/') + 1
+    assert i >= 0
+    head, tail = p[:i], p[i:]
+    if head and head != '/' * len(head):
+        head = head.rstrip('/')
+    return head, tail
+
+
+def exists(path):
+    """Test whether a path exists.  Returns False for broken symbolic links"""
+    try:
+        assert path is not None
+        os.stat(path)
+    except os.error:
+        return False
+    return True
diff --git a/rpython/rlib/test/test_rstacklet.py b/rpython/rlib/test/test_rstacklet.py
--- a/rpython/rlib/test/test_rstacklet.py
+++ b/rpython/rlib/test/test_rstacklet.py
@@ -1,5 +1,6 @@
 import gc, sys
 import py
+import platform
 from rpython.rtyper.tool.rffi_platform import CompilationError
 try:
     from rpython.rlib import rstacklet
@@ -332,6 +333,10 @@
     gc = 'minimark'
     gcrootfinder = 'asmgcc'
 
+    @py.test.mark.skipif("sys.platform != 'linux2' or platform.machine().startswith('arm')")
+    def test_demo1(self):
+        BaseTestStacklet.test_demo1(self)
+
 class TestStackletShadowStack(BaseTestStacklet):
     gc = 'minimark'
     gcrootfinder = 'shadowstack'
diff --git a/rpython/translator/c/src/stacklet/switch_arm_gcc.h b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
--- a/rpython/translator/c/src/stacklet/switch_arm_gcc.h
+++ b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
@@ -1,3 +1,10 @@
+#if __ARM_ARCH__ >= 5
+# define call_reg(x) "blx " #x "\n"
+#elif defined (__ARM_ARCH_4T__)
+# define call_reg(x) "mov lr, pc ; bx " #x "\n"
+#else
+# define call_reg(x) "mov lr, pc ; mov pc, " #x "\n"
+#endif
 
 static void __attribute__((optimize("O3"))) *slp_switch(void *(*save_state)(void*, void*),
                         void *(*restore_state)(void*, void*),
@@ -11,7 +18,7 @@
     "mov r5, %[extra]\n"
     "mov r0, sp\n"        	/* arg 1: current (old) stack pointer */
     "mov r1, r5\n"        	/* arg 2: extra                       */
-    "blx r3\n"				/* call save_state()                  */
+    call_reg(r3)		/* call save_state()                  */
 
     /* skip the rest if the return value is null */
     "cmp r0, #0\n"
@@ -23,7 +30,7 @@
 	stack is not restored yet.  It contains only garbage here. */
     "mov r1, r5\n"       	/* arg 2: extra                       */
    	 						/* arg 1: current (new) stack pointer is already in r0*/
-    "blx r4\n"           	/* call restore_state()               */
+    call_reg(r4)		/* call restore_state()               */
 
     /* The stack's content is now restored. */
     "zero:\n"

From noreply at buildbot.pypy.org  Mon Oct  7 17:27:37 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  7 Oct 2013 17:27:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: more copy-paste from os.path
Message-ID: <20131007152737.A34FB1DAE5B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67184:5558255bbd7c
Date: 2013-10-07 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/5558255bbd7c/

Log:	more copy-paste from os.path

diff --git a/rpython/rlib/rpath.py b/rpython/rlib/rpath.py
--- a/rpython/rlib/rpath.py
+++ b/rpython/rlib/rpath.py
@@ -61,3 +61,65 @@
     except os.error:
         return False
     return True
+
+
+import os
+from os.path import isabs, islink, abspath, normpath
+
+def join(a, p):
+    """Join two or more pathname components, inserting '/' as needed.
+    If any component is an absolute path, all previous path components
+    will be discarded.  An empty last part will result in a path that
+    ends with a separator."""
+    path = a
+    for b in p:
+        if b.startswith('/'):
+            path = b
+        elif path == '' or path.endswith('/'):
+            path +=  b
+        else:
+            path += '/' + b
+    return path
+
+def realpath(filename):
+    """Return the canonical path of the specified filename, eliminating any
+symbolic links encountered in the path."""
+    if isabs(filename):
+        bits = ['/'] + filename.split('/')[1:]
+    else:
+        bits = [''] + filename.split('/')
+
+    for i in range(2, len(bits)+1):
+        component = join(bits[0], bits[1:i])
+        # Resolve symbolic links.
+        if islink(component):
+            resolved = _resolve_link(component)
+            if resolved is None:
+                # Infinite loop -- return original component + rest of the path
+                return abspath(join(component, bits[i:]))
+            else:
+                newpath = join(resolved, bits[i:])
+                return realpath(newpath)
+
+    return abspath(filename)
+
+
+def _resolve_link(path):
+    """Internal helper function.  Takes a path and follows symlinks
+    until we either arrive at something that isn't a symlink, or
+    encounter a path we've seen before (meaning that there's a loop).
+    """
+    paths_seen = {}
+    while islink(path):
+        if path in paths_seen:
+            # Already seen this path, so we must have a symlink loop
+            return None
+        paths_seen[path] = None
+        # Resolve where the link points to
+        resolved = os.readlink(path)
+        if not isabs(resolved):
+            dir = dirname(path)
+            path = normpath(join(dir, [resolved]))
+        else:
+            path = normpath(resolved)
+    return path

From noreply at buildbot.pypy.org  Mon Oct  7 17:35:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  7 Oct 2013 17:35:00 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Make minimarkpage.py support
 optionally incremental mass freeing.
Message-ID: <20131007153500.ABE2B1DAE5B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67185:4ac5aa29679e
Date: 2013-10-07 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/4ac5aa29679e/

Log:	Make minimarkpage.py support optionally incremental mass freeing.

diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -1,3 +1,4 @@
+import sys
 from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, rffi
 from rpython.rlib.rarithmetic import LONG_BIT, r_uint
 from rpython.rlib.objectmodel import we_are_translated
@@ -99,12 +100,10 @@
         # a pointer to a page that has room for at least one more
         # allocation of the given size.
         length = small_request_threshold / WORD + 1
-        self.page_for_size = lltype.malloc(rffi.CArray(PAGE_PTR), length,
-                                           flavor='raw', zero=True,
-                                           immortal=True)
-        self.full_page_for_size = lltype.malloc(rffi.CArray(PAGE_PTR), length,
-                                                flavor='raw', zero=True,
-                                                immortal=True)
+        self.page_for_size          = self._new_page_ptr_list(length)
+        self.full_page_for_size     = self._new_page_ptr_list(length)
+        self.old_page_for_size      = self._new_page_ptr_list(length)
+        self.old_full_page_for_size = self._new_page_ptr_list(length)
         self.nblocks_for_size = lltype.malloc(rffi.CArray(lltype.Signed),
                                               length, flavor='raw',
                                               immortal=True)
@@ -143,6 +142,12 @@
         self.total_memory_used = r_uint(0)
 
 
+    def _new_page_ptr_list(self, length):
+        return lltype.malloc(rffi.CArray(PAGE_PTR), length,
+                             flavor='raw', zero=True,
+                             immortal=True)
+
+
     def malloc(self, size):
         """Allocate a block from a page in an arena."""
         nsize = llmemory.raw_malloc_usage(size)
@@ -248,9 +253,7 @@
                 arena = arena.nextarena
 
 
-    def allocate_new_arena(self):
-        """Loads in self.current_arena the arena to allocate from next."""
-        #
+    def _pick_next_arena(self):
         # Pick an arena from 'arenas_lists[i]', with i as small as possible
         # but > 0.  Use caching with 'min_empty_nfreepages', which guarantees
         # that 'arenas_lists[1:min_empty_nfreepages]' are all empty.
@@ -262,10 +265,25 @@
                 # Found it.
                 self.current_arena = self.arenas_lists[i]
                 self.arenas_lists[i] = self.current_arena.nextarena
-                return
+                return True
             #
             i += 1
             self.min_empty_nfreepages = i
+        return False
+
+
+    def allocate_new_arena(self):
+        """Loads in self.current_arena the arena to allocate from next."""
+        #
+        if self._pick_next_arena():
+            return
+        #
+        # Maybe we are incrementally collecting, in which case an arena
+        # could have more free pages thrown into it than arenas_lists[]
+        # account for.  Rehash and retry.
+        self._rehash_arenas_lists()
+        if self._pick_next_arena():
+            return
         #
         # No more arena with any free page.  We must allocate a new arena.
         if not we_are_translated():
@@ -297,14 +315,32 @@
     allocate_new_arena._dont_inline_ = True
 
 
-    def mass_free(self, ok_to_free_func):
-        """For each object, if ok_to_free_func(obj) returns True, then free
-        the object.
+    def mass_free_prepare(self):
+        """Prepare calls to mass_free_incremental(): moves the chained lists
+        into 'self.old_xxx'.
         """
         self.total_memory_used = r_uint(0)
         #
-        # For each size class:
         size_class = self.small_request_threshold >> WORD_POWER_2
+        self.size_class_with_old_pages = size_class
+        #
+        while size_class >= 1:
+            self.old_page_for_size[size_class]      = (
+                            self.page_for_size[size_class])
+            self.old_full_page_for_size[size_class] = (
+                            self.full_page_for_size[size_class])
+            self.page_for_size[size_class]      = PAGE_NULL
+            self.full_page_for_size[size_class] = PAGE_NULL
+            size_class -= 1
+
+
+    def mass_free_incremental(self, ok_to_free_func, max_pages):
+        """For each object, if ok_to_free_func(obj) returns True, then free
+        the object.  This returns True if complete, or False if the limit
+        'max_pages' is reached.
+        """
+        size_class = self.size_class_with_old_pages
+        #
         while size_class >= 1:
             #
             # Walk the pages in 'page_for_size[size_class]' and
@@ -313,10 +349,30 @@
             # and become available for reuse by any size class.  Pages
             # not completely freed are re-chained either in
             # 'full_page_for_size[]' or 'page_for_size[]'.
-            self.mass_free_in_pages(size_class, ok_to_free_func)
+            max_pages = self.mass_free_in_pages(size_class, ok_to_free_func,
+                                                max_pages)
+            if max_pages <= 0:
+                self.size_class_with_old_pages = size_class
+                return False
             #
             size_class -= 1
         #
+        self._rehash_arenas_lists()
+        return True
+
+
+    def mass_free(self, ok_to_free_func):
+        """For each object, if ok_to_free_func(obj) returns True, then free
+        the object.
+        """
+        self.mass_free_prepare()
+        #
+        res = self.mass_free_incremental(ok_to_free_func, sys.maxint)
+        ll_assert(res, "non-incremental mass_free_in_pages() returned False")
+
+
+    def _rehash_arenas_lists(self):
+        #
         # Rehash arenas into the correct arenas_lists[i].  If
         # 'self.current_arena' contains an arena too, it remains there.
         (self.old_arenas_lists, self.arenas_lists) = (
@@ -353,18 +409,20 @@
         self.min_empty_nfreepages = 1
 
 
-    def mass_free_in_pages(self, size_class, ok_to_free_func):
+    def mass_free_in_pages(self, size_class, ok_to_free_func, max_pages):
         nblocks = self.nblocks_for_size[size_class]
         block_size = size_class * WORD
-        remaining_partial_pages = PAGE_NULL
-        remaining_full_pages = PAGE_NULL
+        remaining_partial_pages = self.page_for_size[size_class]
+        remaining_full_pages = self.full_page_for_size[size_class]
         #
         step = 0
         while step < 2:
             if step == 0:
-                page = self.full_page_for_size[size_class]
+                page = self.old_full_page_for_size[size_class]
+                self.old_full_page_for_size[size_class] = PAGE_NULL
             else:
-                page = self.page_for_size[size_class]
+                page = self.old_page_for_size[size_class]
+                self.old_page_for_size[size_class] = PAGE_NULL
             #
             while page != PAGE_NULL:
                 #
@@ -392,12 +450,26 @@
                     # No object survives; free the page.
                     self.free_page(page)
 
+                #
+                max_pages -= 1
+                if max_pages <= 0:
+                    # End of the incremental step: store back the unprocessed
+                    # pages into self.old_xxx and return early
+                    if step == 0:
+                        self.old_full_page_for_size[size_class] = nextpage
+                    else:
+                        self.old_page_for_size[size_class] = nextpage
+                    step = 99     # stop
+                    break
+
                 page = nextpage
             #
-            step += 1
+            else:
+                step += 1
         #
         self.page_for_size[size_class] = remaining_partial_pages
         self.full_page_for_size[size_class] = remaining_full_pages
+        return max_pages
 
 
     def free_page(self, page):
diff --git a/rpython/memory/gc/test/test_minimarkpage.py b/rpython/memory/gc/test/test_minimarkpage.py
--- a/rpython/memory/gc/test/test_minimarkpage.py
+++ b/rpython/memory/gc/test/test_minimarkpage.py
@@ -402,7 +402,7 @@
 
 # ____________________________________________________________
 
-def test_random():
+def test_random(incremental=False):
     import random
     pagesize = hdrsize + 24*WORD
     num_pages = 3
@@ -428,30 +428,52 @@
                 raise DoneTesting
         a.mark_freed = my_mark_freed
     ac.allocate_new_arena = my_allocate_new_arena
+
+    def allocate_object(live_objects):
+        size_class = random.randrange(1, 7)
+        obj = ac.malloc(size_class * WORD)
+        at = (obj.arena, obj.offset)
+        assert at not in live_objects
+        live_objects[at] = size_class * WORD
+
     try:
         while True:
             #
             # Allocate some more objects
             for i in range(random.randrange(50, 100)):
-                size_class = random.randrange(1, 7)
-                obj = ac.malloc(size_class * WORD)
-                at = (obj.arena, obj.offset)
-                assert at not in live_objects
-                live_objects[at] = size_class * WORD
+                allocate_object(live_objects)
             #
             # Free half the objects, randomly
             ok_to_free = OkToFree(ac, lambda obj: random.random() < 0.5,
                                   multiarenas=True)
-            ac.mass_free(ok_to_free)
+            live_objects_extra = {}
+            fresh_extra = 0
+            if not incremental:
+                ac.mass_free(ok_to_free)
+            else:
+                ac.mass_free_prepare()
+                while not ac.mass_free_incremental(ok_to_free,
+                                                   random.randrange(1, 3)):
+                    print '[]'
+                    prev = ac.total_memory_used
+                    allocate_object(live_objects_extra)
+                    fresh_extra += ac.total_memory_used - prev
             #
             # Check that we have seen all objects
             assert sorted(ok_to_free.seen) == sorted(live_objects)
-            surviving_total_size = 0
+            surviving_total_size = fresh_extra
             for at, freed in ok_to_free.seen.items():
                 if freed:
                     del live_objects[at]
                 else:
                     surviving_total_size += live_objects[at]
             assert ac.total_memory_used == surviving_total_size
+            #
+            assert not (set(live_objects) & set(live_objects_extra))
+            live_objects.update(live_objects_extra)
+            #
     except DoneTesting:
         pass
+
+def test_random_incremental():
+    test_random(incremental=True)

From noreply at buildbot.pypy.org  Mon Oct  7 21:20:14 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon,  7 Oct 2013 21:20:14 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20131007192014.2288F1C00EC@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67186:7cd5f50e628b
Date: 2013-10-07 12:18 -0700
http://bitbucket.org/pypy/pypy/changeset/7cd5f50e628b/

Log:	merge default

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -363,9 +363,11 @@
     pass
 
 
-def connect(database, **kwargs):
-    factory = kwargs.get("factory", Connection)
-    return factory(database, **kwargs)
+def connect(database, timeout=5.0, detect_types=0, isolation_level="",
+                 check_same_thread=True, factory=None, cached_statements=100):
+    factory = Connection if not factory else factory
+    return factory(database, timeout, detect_types, isolation_level,
+                    check_same_thread, factory, cached_statements)
 
 
 def _unicode_text_factory(x):
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -22,9 +22,11 @@
         return result
 
     elif value.typePtr == typeCache.BooleanType:
-        return result
+        return bool(value.internalRep.longValue)
     elif value.typePtr == typeCache.ByteArrayType:
-        return result
+        size = tkffi.new('int*')
+        data = tklib.Tcl_GetByteArrayFromObj(value, size)
+        return tkffi.buffer(data, size[0])[:]
     elif value.typePtr == typeCache.DoubleType:
         return value.internalRep.doubleValue
     elif value.typePtr == typeCache.IntType:
@@ -44,7 +46,7 @@
             result.append(FromObj(app, tcl_elem[0]))
         return tuple(result)
     elif value.typePtr == typeCache.ProcBodyType:
-        return result
+        pass  # fall through and return tcl object.
     elif value.typePtr == typeCache.StringType:
         buf = tklib.Tcl_GetUnicode(value)
         length = tklib.Tcl_GetCharLength(value)
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -72,6 +72,7 @@
 int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
 char *Tcl_GetString(Tcl_Obj* objPtr);
 char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+unsigned char *Tcl_GetByteArrayFromObj(Tcl_Obj* objPtr, int* lengthPtr);
 
 Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
 int Tcl_GetCharLength(Tcl_Obj* objPtr);
diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst
--- a/pypy/doc/arm.rst
+++ b/pypy/doc/arm.rst
@@ -35,6 +35,11 @@
   * ``qemu-system``
   * ``qemu-user-static``
 
+- The dependencies above are in addition to the ones needed for a regular
+  translation, `listed here`_.
+
+.. _`listed here`: getting-started-python.html#translating-the-pypy-python-interpreter
+
 
 Creating a Qemu based ARM chroot
 --------------------------------
diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -107,9 +107,15 @@
         for i in range(min(len(varnames), self.getfastscopelength())):
             name = varnames[i]
             w_value = fastscope_w[i]
+            w_name = self.space.wrap(name.decode('utf-8'))
             if w_value is not None:
-                w_name = self.space.wrap(name.decode('utf-8'))
                 self.space.setitem(self.w_locals, w_name, w_value)
+            else:
+                try:
+                    self.space.delitem(self.w_locals, w_name)
+                except OperationError as e:
+                    if not e.match(self.space, self.space.w_KeyError):
+                        raise
 
     def locals2fast(self):
         # Copy values from self.w_locals to the fastlocals
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1073,6 +1073,8 @@
 
         sys.path.append(self.goal_dir)
         # make sure cwd does not contain a stdlib
+        if self.tmp_dir.startswith(self.trunkdir):
+            skip('TMPDIR is inside the PyPy source')
         os.chdir(self.tmp_dir)
         tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c')
         try:
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -129,10 +129,21 @@
     def test_locals(self):
         def f():
             return locals()
+
         def g(c=0, b=0, a=0):
             return locals()
+
         assert f() == {}
-        assert g() == {'a':0, 'b':0, 'c':0}
+        assert g() == {'a': 0, 'b': 0, 'c': 0}
+
+    def test_locals_deleted_local(self):
+        def f():
+            a = 3
+            locals()
+            del a
+            return locals()
+
+        assert f() == {}
 
     def test_dir(self):
         def f():
@@ -298,22 +309,6 @@
         assert next(x) == 3
 
     def test_range_args(self):
-##        # range() attributes are deprecated and were removed in Python 2.3.
-##        x = range(2)
-##        assert x.start == 0
-##        assert x.stop == 2
-##        assert x.step == 1
-
-##        x = range(2,10,2)
-##        assert x.start == 2
-##        assert x.stop == 10
-##        assert x.step == 2
-
-##        x = range(2.3, 10.5, 2.4)
-##        assert x.start == 2
-##        assert x.stop == 10
-##        assert x.step == 2
-
         raises(ValueError, range, 0, 1, 0)
 
     def test_range_repr(self):
@@ -374,7 +369,7 @@
         raises(TypeError, range, 1, 3+2j)
         raises(TypeError, range, 1, 2, '1')
         raises(TypeError, range, 1, 2, 3+2j)
-    
+
     def test_sorted(self):
         l = []
         sorted_l = sorted(l)
@@ -393,7 +388,7 @@
         assert sorted_l is not l
         assert sorted_l == ['C', 'b', 'a']
         raises(TypeError, sorted, [], reverse=None)
-        
+
     def test_reversed_simple_sequences(self):
         l = range(5)
         rev = reversed(l)
@@ -409,7 +404,7 @@
                 return 42
         obj = SomeClass()
         assert reversed(obj) == 42
-    
+
     def test_return_None(self):
         class X(object): pass
         x = X()
@@ -465,7 +460,7 @@
         assert eval("1+2") == 3
         assert eval(" \t1+2\n") == 3
         assert eval("len([])") == 0
-        assert eval("len([])", {}) == 0        
+        assert eval("len([])", {}) == 0
         # cpython 2.4 allows this (raises in 2.3)
         assert eval("3", None, None) == 3
         i = 4
diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -149,14 +149,17 @@
          only used if the array is constructed that way. Almost always this
          parameter is NULL.
     """
-    if min_depth !=0 or max_depth != 0:
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            '_PyArray_FromAny called with not-implemented min_dpeth or max_depth argument'))
     if requirements not in (0, NPY_DEFAULT):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             '_PyArray_FromAny called with not-implemented requirements argument'))
     w_array = array(space, w_obj, w_dtype=w_dtype, copy=False)
-    if w_array.is_scalar():
+    if min_depth !=0 and len(w_array.get_shape()) < min_depth:
+        raise OperationError(space.w_ValueError, space.wrap(
+            'object of too small depth for desired array'))
+    elif max_depth !=0 and len(w_array.get_shape()) > max_depth:
+        raise OperationError(space.w_ValueError, space.wrap(
+            'object of too deep for desired array'))
+    elif w_array.is_scalar():
         # since PyArray_DATA() fails on scalars, create a 1D array and set empty
         # shape. So the following combination works for *reading* scalars:
         #     PyObject *arr = PyArray_FromAny(obj);
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -90,15 +90,16 @@
     def test_FromAny(self, space, api):
         a = array(space, [10, 5, 3])
         assert api._PyArray_FromAny(a, NULL, 0, 0, 0, NULL) is a
-        self.raises(space, api, NotImplementedError, api._PyArray_FromAny,
-                    a, NULL, 0, 3, 0, NULL)
+        assert api._PyArray_FromAny(a, NULL, 1, 4, 0, NULL) is a
+        self.raises(space, api, ValueError, api._PyArray_FromAny,
+                    a, NULL, 4, 5, 0, NULL)
 
     def test_FromObject(self, space, api):
         a = array(space, [10, 5, 3])
         assert api._PyArray_FromObject(a, a.get_dtype().num, 0, 0) is a
-        exc = self.raises(space, api, NotImplementedError, api._PyArray_FromObject,
-                    a, 11, 0, 3)
-        assert exc.errorstr(space).find('FromObject') >= 0
+        exc = self.raises(space, api, ValueError, api._PyArray_FromObject,
+                    a, 11, 4, 5)
+        assert exc.errorstr(space).find('desired') >= 0
 
     def test_list_from_fixedptr(self, space, api):
         A = lltype.GcArray(lltype.Float)
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -350,9 +350,8 @@
             v = hi
         del partials[added:]
         if v != 0.0:
-            if rfloat.isinf(v) or rfloat.isnan(v):
-                if (not rfloat.isinf(original) and
-                    not rfloat.isnan(original)):
+            if not rfloat.isfinite(v):
+                if rfloat.isfinite(original):
                     raise OperationError(space.w_OverflowError,
                                          space.wrap("intermediate overflow"))
                 if rfloat.isinf(original):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2966,6 +2966,12 @@
 
         assert len(list(a[0])) == 2
 
+    def test_issue_1589(self):
+        import numpypy as numpy
+        c = numpy.array([[(1, 2, 'a'), (3, 4, 'b')], [(5, 6, 'c'), (7, 8, 'd')]],
+                        dtype=[('bg', 'i8'), ('fg', 'i8'), ('char', 'S1')])
+        assert c[0][0]["char"] == 'a'
+
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1766,14 +1766,14 @@
 
     def store(self, arr, i, offset, box):
         assert isinstance(box, interp_boxes.W_StringBox)
-        # XXX simplify to range(box.dtype.get_size()) ?
         return self._store(arr.storage, i, offset, box)
 
     @jit.unroll_safe
     def _store(self, storage, i, offset, box):
         assert isinstance(box, interp_boxes.W_StringBox)
-        for k in range(min(self.size, box.arr.size-offset)):
-            storage[k + i] = box.arr.storage[k + offset]
+        # XXX simplify to range(box.dtype.get_size()) ?
+        for k in range(min(self.size, box.arr.size-box.ofs)):
+            storage[k + offset + i] = box.arr.storage[k + box.ofs]
 
     def read(self, arr, i, offset, dtype=None):
         if dtype is None:
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -277,3 +277,28 @@
             f1 = call_release_gil(..., descr=<Calli 4 ii EF=6 OS=62>)
             ...
         """)
+
+    def test__cffi_bug1(self):
+        from rpython.rlib.test.test_clibffi import get_libm_name
+        def main(libm_name):
+            try:
+                import _cffi_backend
+            except ImportError:
+                sys.stderr.write('SKIP: cannot import _cffi_backend\n')
+                return 0
+
+            libm = _cffi_backend.load_library(libm_name)
+            BDouble = _cffi_backend.new_primitive_type("double")
+            BSin = _cffi_backend.new_function_type([BDouble], BDouble)
+            sin = libm.load_function(BSin, 'sin')
+
+            def f(*args):
+                for i in range(300):
+                    sin(*args)
+
+            f(1.0)
+            f(1)
+        #
+        libm_name = get_libm_name(sys.platform)
+        log = self.run(main, [libm_name])
+        # assert did not crash
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -113,7 +113,7 @@
             i13 = strgetitem(p9, 0)
             i15 = int_eq(i13, 45)
             guard_false(i15, descr=...)
-            i17 = int_sub(0, i10)
+            i17 = int_neg(i10)
             i19 = int_gt(i10, 23)
             guard_false(i19, descr=...)
             p21 = newstr(23)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -1,4 +1,3 @@
-import py
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
 
@@ -51,7 +50,6 @@
         """)
 
     def test_lock_acquire_release(self):
-        py.test.skip("test too precise, please fix me")
         def main(n):
             import threading
             lock = threading.Lock()
@@ -62,35 +60,30 @@
         assert log.result == main(500)
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-        i58 = int_gt(i43, 0)
-        guard_true(i58, descr=<Guard0x10483adb8>)
-        p59 = getfield_gc(p15, descr=<FieldP pypy.module.thread.os_lock.Lock.inst_lock 8>)
-        i60 = getfield_gc(p59, descr=<FieldU rpython.rlib.rthread.Lock.inst__lock 8>)
+        i55 = int_gt(i43, 0)
+        guard_true(i55, descr=...)
+        p56 = force_token()
+        setfield_gc(p0, p56, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        i57 = call_release_gil(..., i36, 1, descr=<Calli 4 ii EF=6>)
+        guard_not_forced(descr=...)
+        guard_no_exception(descr=...)
+        i58 = int_is_true(i57)
+        guard_true(i58, descr=...)
+        i59 = int_sub(i43, 1)
+        guard_not_invalidated(descr=...)
         p61 = force_token()
-        setfield_gc(p0, p61, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 24>)
-        i62 = call_release_gil(4312440032, i60, 1, descr=<Calli 4 ii EF=6>)
-        guard_not_forced(descr=<Guard0x103f3cca0>)
-        guard_no_exception(descr=<Guard0x10483ad40>)
+        setfield_gc(p0, p61, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        i62 = call_release_gil(..., i36, 0, descr=<Calli 4 ii EF=6>)
+        guard_not_forced(descr=...)
+        guard_no_exception(descr=...)
         i63 = int_is_true(i62)
-        guard_true(i63, descr=<Guard0x10483acc8>)
-        i64 = int_sub(i43, 1)
-        guard_not_invalidated(descr=<Guard0x10483ac50>)
-        p66 = getfield_gc(p15, descr=<FieldP pypy.module.thread.os_lock.Lock.inst_lock 8>)
-        i67 = getfield_gc(p66, descr=<FieldU rpython.rlib.rthread.Lock.inst__lock 8>)
-        p68 = force_token()
-        setfield_gc(p0, p68, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 24>)
-        i69 = call_release_gil(4312440032, i67, 0, descr=<Calli 4 ii EF=6>)
-        guard_not_forced(descr=<Guard0x103f3cc20>)
-        guard_no_exception(descr=<Guard0x10483aae8>)
-        i70 = int_is_true(i69)
-        guard_false(i70, descr=<Guard0x10483aa70>)
-        i71 = getfield_gc(p66, descr=<FieldU rpython.rlib.rthread.Lock.inst__lock 8>)
-        p72 = force_token()
-        setfield_gc(p0, p72, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 24>)
-        call_release_gil(4312441056, i71, descr=<Callv 0 i EF=6>)
-        guard_not_forced(descr=<Guard0x103f3cba0>)
-        guard_no_exception(descr=<Guard0x10483a9f8>)
-        guard_not_invalidated(descr=<Guard0x10483a980>)
+        guard_false(i63, descr=...)
+        p64 = force_token()
+        setfield_gc(p0, p64, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        call_release_gil(..., i36, descr=<Callv 0 i EF=6>)
+        guard_not_forced(descr=...)
+        guard_no_exception(descr=...)
+        guard_not_invalidated(descr=...)
         --TICK--
-        jump(..., descr=TargetToken(4361239720))
+        jump(..., descr=...)
         """)
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -31,6 +31,12 @@
     result = list(cursor)
     assert result == [(42,)]
 
+def test_connect_takes_same_positional_args_as_Connection(con):
+    from inspect import getargspec
+    clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
+    conargs = getargspec(_sqlite3.connect).args
+    assert clsargs == conargs
+
 def test_total_changes_after_close(con):
     con.close()
     pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -75,6 +75,8 @@
 class Lock(W_Root):
     "A wrappable box around an interp-level lock object."
 
+    _immutable_fields_ = ["lock"]
+
     def __init__(self, space):
         self.space = space
         try:
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -398,21 +398,24 @@
     x = w_float1.floatval
     y = w_float2.floatval
 
+    return W_FloatObject(_pow(space, x, y))
+
+def _pow(space, x, y):
     # Sort out special cases here instead of relying on pow()
-    if y == 2.0:                      # special case for performance:
-        return W_FloatObject(x * x)   # x * x is always correct
+    if y == 2.0:       # special case for performance:
+        return x * x   # x * x is always correct
     if y == 0.0:
         # x**0 is 1, even 0**0
-        return W_FloatObject(1.0)
+        return 1.0
     if isnan(x):
         # nan**y = nan, unless y == 0
-        return W_FloatObject(x)
+        return x
     if isnan(y):
         # x**nan = nan, unless x == 1; x**nan = x
         if x == 1.0:
-            return W_FloatObject(1.0)
+            return 1.0
         else:
-            return W_FloatObject(y)
+            return y
     if isinf(y):
         # x**inf is: 0.0 if abs(x) < 1; 1.0 if abs(x) == 1; inf if
         # abs(x) > 1 (including case where x infinite)
@@ -421,11 +424,11 @@
         # abs(x) > 1 (including case where v infinite)
         x = abs(x)
         if x == 1.0:
-            return W_FloatObject(1.0)
+            return 1.0
         elif (y > 0.0) == (x > 1.0):
-            return W_FloatObject(INFINITY)
+            return INFINITY
         else:
-            return W_FloatObject(0.0)
+            return 0.0
     if isinf(x):
         # (+-inf)**w is: inf for w positive, 0 for w negative; in oth
         # cases, we need to add the appropriate sign if w is an odd
@@ -433,14 +436,14 @@
         y_is_odd = math.fmod(abs(y), 2.0) == 1.0
         if y > 0.0:
             if y_is_odd:
-                return W_FloatObject(x)
+                return x
             else:
-                return W_FloatObject(abs(x))
+                return abs(x)
         else:
             if y_is_odd:
-                return W_FloatObject(copysign(0.0, x))
+                return copysign(0.0, x)
             else:
-                return W_FloatObject(0.0)
+                return 0.0
 
     if x == 0.0:
         if y < 0.0:
@@ -454,7 +457,7 @@
     # -           pipermail/python-bugs-list/2003-March/016795.html
     if x < 0.0:
         if isnan(y):
-            return W_FloatObject(NAN)
+            return NAN
         if math.floor(y) != y:
             # Negative numbers raised to fractional powers become
             # complex
@@ -470,9 +473,9 @@
     if x == 1.0:
         # (-1) ** large_integer also ends up here
         if negate_result:
-            return W_FloatObject(-1.0)
+            return -1.0
         else:
-            return W_FloatObject(1.0)
+            return 1.0
 
     try:
         # We delegate to our implementation of math.pow() the error detection.
@@ -486,7 +489,7 @@
 
     if negate_result:
         z = -z
-    return W_FloatObject(z)
+    return z
 
 
 def neg__Float(space, w_float1):
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -194,10 +194,10 @@
 
 W_AbstractTupleObject.typedef = StdTypeDef(
     "tuple",
-    __doc__ = '''tuple() -> an empty tuple
+    __doc__ = """tuple() -> an empty tuple
 tuple(sequence) -> tuple initialized from sequence's items
 
-If the argument is a tuple, the return value is the same object.''',
+If the argument is a tuple, the return value is the same object.""",
     __new__ = interp2app(W_AbstractTupleObject.descr_new),
     __repr__ = interp2app(W_AbstractTupleObject.descr_repr),
     __hash__ = interpindirect2app(W_AbstractTupleObject.descr_hash),
diff --git a/pypy/pytest-A.py b/pypy/pytest-A.py
--- a/pypy/pytest-A.py
+++ b/pypy/pytest-A.py
@@ -5,7 +5,6 @@
     'arm': ['interpreter/astcompiler/test',
             'interpreter/pyparser/test',
             'interpreter/test',
-            'interpreter/test2',
             'module/test_lib_pypy',
             'objspace/std/test',
     ],
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -1063,7 +1063,6 @@
         assert len(graph.startblock.exits) == 1
         assert graph.startblock.exits[0].target == graph.returnblock
 
-
     def test_global_variable(self):
         def global_var_missing():
             return a
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -381,6 +381,8 @@
             res = self.llinterp.eval_graph(ptr._obj.graph, args)
         else:
             res = ptr._obj._callable(*args)
+        if RESULT is lltype.Void:
+            return None
         return support.cast_result(RESULT, res)
 
     def _do_call(self, func, args_i, args_r, args_f, calldescr):
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -87,9 +87,11 @@
     def _escape(self, box):
         if box in self.new_boxes:
             self.new_boxes[box] = False
-        if box in self.dependencies:
-            deps = self.dependencies[box]
-            del self.dependencies[box]
+        try:
+            deps = self.dependencies.pop(box)
+        except KeyError:
+            pass
+        else:
             for dep in deps:
                 self._escape(dep)
 
@@ -117,15 +119,18 @@
             # effects are so well defined.
             elif effectinfo.oopspecindex == effectinfo.OS_ARRAYCOPY:
                 # The destination box
-                if argboxes[2] in self.new_boxes:
-                    # XXX: no descr here so we invalidate any of them, not just
-                    # of the correct type
-                    # XXX: in theory the indices of the copy could be looked at
-                    # as well
-                    for descr, cache in self.heap_array_cache.iteritems():
+                if (
+                    argboxes[2] in self.new_boxes and
+                    len(effectinfo.write_descrs_arrays) == 1
+                ):
+                    # Fish the descr out of the effectinfo
+                    cache = self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
+                    if cache is not None:
+                        # XXX: in theory the indices of the copy could be
+                        # looked at
                         for idx, cache in cache.iteritems():
                             for frombox in cache.keys():
-                                if not self.new_boxes.get(frombox, False):
+                                if not self.is_unescaped(frombox):
                                     del cache[frombox]
                     return
             else:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -107,6 +107,11 @@
         v2 = self.getvalue(op.getarg(1))
         if v2.is_constant() and v2.box.getint() == 0:
             self.make_equal_to(op.result, v1)
+        elif v1.is_constant() and v1.box.getint() == 0:
+            op = op.copy_and_change(rop.INT_NEG, args=[v2.box])
+            self.emit_operation(op)
+        elif v1 is v2:
+            self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
             # Synthesize the reverse ops for optimize_default to reuse
@@ -166,6 +171,8 @@
 
         if v2.is_constant() and v2.box.getint() == 0:
             self.make_equal_to(op.result, v1)
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
 
@@ -175,6 +182,8 @@
 
         if v2.is_constant() and v2.box.getint() == 0:
             self.make_equal_to(op.result, v1)
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -45,7 +45,7 @@
                 return self.optimize_JUMP(op.copy_and_change(rop.JUMP))
             self.last_label_descr = op.getdescr()
         self.emit_operation(op)
-        
+
     def optimize_JUMP(self, op):
         if not self.unroll:
             descr = op.getdescr()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -3735,6 +3735,33 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_sub_identity(self):
+        ops = """
+        [i0]
+        i1 = int_sub(i0, i0)
+        i2 = int_sub(i1, i0)
+        jump(i1, i2)
+        """
+        expected = """
+        [i0]
+        i2 = int_neg(i0)
+        jump(0, i2)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_shift_zero(self):
+        ops = """
+        [i0]
+        i1 = int_lshift(0, i0)
+        i2 = int_rshift(0, i0)
+        jump(i1, i2)
+        """
+        expected = """
+        [i0]
+        jump(0, 0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_bound_and(self):
         ops = """
         [i0]
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -45,6 +45,15 @@
             return value
         return OptValue(self.force_box(optforce))
 
+    def get_args_for_fail(self, modifier):
+        # checks for recursion: it is False unless
+        # we have already seen the very same keybox
+        if self.box is None and not modifier.already_seen_virtual(self.keybox):
+            self._get_args_for_fail(modifier)
+
+    def _get_args_for_fail(self, modifier):
+        raise NotImplementedError("abstract base")
+
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
             return self._make_virtual(modifier)
@@ -193,16 +202,13 @@
             self._cached_sorted_fields = lst
         return lst
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            # checks for recursion: it is False unless
-            # we have already seen the very same keybox
-            lst = self._get_field_descr_list()
-            fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst]
-            modifier.register_virtual_fields(self.keybox, fieldboxes)
-            for ofs in lst:
-                fieldvalue = self._fields[ofs]
-                fieldvalue.get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        lst = self._get_field_descr_list()
+        fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst]
+        modifier.register_virtual_fields(self.keybox, fieldboxes)
+        for ofs in lst:
+            fieldvalue = self._fields[ofs]
+            fieldvalue.get_args_for_fail(modifier)
 
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
@@ -254,18 +260,15 @@
     def set_item_value(self, i, newval):
         raise NotImplementedError
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            # checks for recursion: it is False unless
-            # we have already seen the very same keybox
-            itemboxes = []
-            for i in range(self.getlength()):
-                itemvalue = self.get_item_value(i)
-                itemboxes.append(itemvalue.get_key_box())
-            modifier.register_virtual_fields(self.keybox, itemboxes)
-            for i in range(self.getlength()):
-                itemvalue = self.get_item_value(i)
-                itemvalue.get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        itemboxes = []
+        for i in range(self.getlength()):
+            itemvalue = self.get_item_value(i)
+            itemboxes.append(itemvalue.get_key_box())
+        modifier.register_virtual_fields(self.keybox, itemboxes)
+        for i in range(self.getlength()):
+            itemvalue = self.get_item_value(i)
+            itemvalue.get_args_for_fail(modifier)
 
 
 class VArrayValue(AbstractVArrayValue):
@@ -370,17 +373,16 @@
             descrs.append(item_descrs)
         return descrs
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            itemdescrs = self._get_list_of_descrs()
-            itemboxes = []
-            for i in range(len(self._items)):
-                for descr in itemdescrs[i]:
-                    itemboxes.append(self._items[i][descr].get_key_box())
-            modifier.register_virtual_fields(self.keybox, itemboxes)
-            for i in range(len(self._items)):
-                for descr in itemdescrs[i]:
-                    self._items[i][descr].get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        itemdescrs = self._get_list_of_descrs()
+        itemboxes = []
+        for i in range(len(self._items)):
+            for descr in itemdescrs[i]:
+                itemboxes.append(self._items[i][descr].get_key_box())
+        modifier.register_virtual_fields(self.keybox, itemboxes)
+        for i in range(len(self._items)):
+            for descr in itemdescrs[i]:
+                self._items[i][descr].get_args_for_fail(modifier)
 
     def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
@@ -481,6 +483,15 @@
     def getitem_raw(self, offset, length, descr):
         return self.rawbuffer_value.getitem_raw(self.offset+offset, length, descr)
 
+    def _get_args_for_fail(self, modifier):
+        box = self.rawbuffer_value.get_key_box()
+        modifier.register_virtual_fields(self.keybox, [box])
+        self.rawbuffer_value.get_args_for_fail(modifier)
+
+    def _make_virtual(self, modifier):
+        return modifier.make_vrawslice(self.offset)
+
+
 class OptVirtualize(optimizer.Optimization):
     "Virtualize objects until they escape."
 
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -284,7 +284,10 @@
         return VArrayStructInfo(arraydescr, fielddescrs)
 
     def make_vrawbuffer(self, size, offsets, descrs):
-        return VRawBufferStateInfo(size, offsets, descrs)
+        return VRawBufferInfo(size, offsets, descrs)
+
+    def make_vrawslice(self, offset):
+        return VRawSliceInfo(offset)
 
     def make_vstrplain(self, is_unicode=False):
         if is_unicode:
@@ -554,10 +557,13 @@
             debug_print("\t\t", str(untag(i)))
 
 
-class VRawBufferStateInfo(AbstractVirtualInfo):
+class VAbstractRawInfo(AbstractVirtualInfo):
     kind = INT
     is_about_raw = True
 
+
+class VRawBufferInfo(VAbstractRawInfo):
+
     def __init__(self, size, offsets, descrs):
         self.size = size
         self.offsets = offsets
@@ -580,6 +586,25 @@
             debug_print("\t\t", str(untag(i)))
 
 
+class VRawSliceInfo(VAbstractRawInfo):
+
+    def __init__(self, offset):
+        self.offset = offset
+
+    @specialize.argtype(1)
+    def allocate_int(self, decoder, index):
+        assert len(self.fieldnums) == 1
+        base_buffer = decoder.decode_int(self.fieldnums[0])
+        buffer = decoder.int_add_const(base_buffer, self.offset)
+        decoder.virtuals_cache.set_int(index, buffer)
+        return buffer
+
+    def debug_prints(self):
+        debug_print("\tvrawsliceinfo", " at ",  compute_unique_id(self))
+        for i in self.fieldnums:
+            debug_print("\t\t", str(untag(i)))
+
+
 class VArrayStructInfo(AbstractVirtualInfo):
     def __init__(self, arraydescr, fielddescrs):
         self.arraydescr = arraydescr
@@ -783,7 +808,8 @@
         v = self.virtuals_cache.get_int(index)
         if not v:
             v = self.rd_virtuals[index]
-            assert v.is_about_raw and isinstance(v, VRawBufferStateInfo)
+            ll_assert(bool(v), "resume.py: null rd_virtuals[index]")
+            assert v.is_about_raw and isinstance(v, VAbstractRawInfo)
             v = v.allocate_int(self, index)
             ll_assert(v == self.virtuals_cache.get_int(index), "resume.py: bad cache")
         return v
@@ -1116,6 +1142,10 @@
     def write_a_float(self, index, box):
         self.boxes_f[index] = box
 
+    def int_add_const(self, intbox, offset):
+        return self.metainterp.execute_and_record(rop.INT_ADD, None, intbox,
+                                                  ConstInt(offset))
+
 # ---------- when resuming for blackholing, get direct values ----------
 
 def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage,
@@ -1407,6 +1437,9 @@
     def write_a_float(self, index, float):
         self.blackholeinterp.setarg_f(index, float)
 
+    def int_add_const(self, base, offset):
+        return base + offset
+
 # ____________________________________________________________
 
 def dump_storage(storage, liveboxes):
diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py
--- a/rpython/jit/metainterp/test/test_fficall.py
+++ b/rpython/jit/metainterp/test/test_fficall.py
@@ -86,15 +86,17 @@
             data = rffi.ptradd(exchange_buffer, ofs)
             rffi.cast(lltype.Ptr(TYPE), data)[0] = write_rvalue
 
-        def f():
+        def f(i):
             exbuf = lltype.malloc(rffi.CCHARP.TO, (len(avalues)+2) * 16,
-                                  flavor='raw', zero=True)
-            ofs = 16
+                                  flavor='raw')
+
+            targetptr = rffi.ptradd(exbuf, 16)
             for avalue in unroll_avalues:
                 TYPE = rffi.CArray(lltype.typeOf(avalue))
-                data = rffi.ptradd(exbuf, ofs)
-                rffi.cast(lltype.Ptr(TYPE), data)[0] = avalue
-                ofs += 16
+                if i >= 9:    # a guard that can fail
+                    pass
+                rffi.cast(lltype.Ptr(TYPE), targetptr)[0] = avalue
+                targetptr = rffi.ptradd(targetptr, 16)
 
             jit_ffi_call(cif_description, func_addr, exbuf)
 
@@ -102,8 +104,7 @@
                 res = 654321
             else:
                 TYPE = rffi.CArray(lltype.typeOf(rvalue))
-                data = rffi.ptradd(exbuf, ofs)
-                res = rffi.cast(lltype.Ptr(TYPE), data)[0]
+                res = rffi.cast(lltype.Ptr(TYPE), targetptr)[0]
             lltype.free(exbuf, flavor='raw')
             if lltype.typeOf(res) is lltype.SingleFloat:
                 res = float(res)
@@ -117,9 +118,9 @@
             return res == rvalue
 
         with FakeFFI(fake_call_impl_any):
-            res = f()
+            res = f(-42)
             assert matching_result(res, rvalue)
-            res = self.interp_operations(f, [],
+            res = self.interp_operations(f, [-42],
                             supports_floats = supports_floats,
                           supports_longlong = supports_longlong,
                       supports_singlefloats = supports_singlefloats)
@@ -132,6 +133,19 @@
             self.check_operations_history(call_may_force=0,
                                           call_release_gil=expected_call_release_gil)
 
+            ##################################################
+            driver = jit.JitDriver(reds=['i'], greens=[])
+            def main():
+                i = 0
+                while 1:
+                    driver.jit_merge_point(i=i)
+                    res = f(i)
+                    i += 1
+                    if i == 12:
+                        return res
+            self.meta_interp(main, [])
+
+
     def test_simple_call_int(self):
         self._run([types.signed] * 2, types.signed, [456, 789], -42)
 
diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -29,17 +29,24 @@
 
     OS_ARRAYCOPY = 0
 
-    def __init__(self, extraeffect, oopspecindex):
+    def __init__(self, extraeffect, oopspecindex, write_descrs_arrays):
         self.extraeffect = extraeffect
         self.oopspecindex = oopspecindex
+        self.write_descrs_arrays = write_descrs_arrays
+
 
 class FakeCallDescr(object):
-    def __init__(self, extraeffect, oopspecindex=None):
+    def __init__(self, extraeffect, oopspecindex=None, write_descrs_arrays=[]):
         self.extraeffect = extraeffect
         self.oopspecindex = oopspecindex
+        self.write_descrs_arrays = write_descrs_arrays
 
     def get_extra_info(self):
-        return FakeEffectinfo(self.extraeffect, self.oopspecindex)
+        return FakeEffectinfo(
+            self.extraeffect, self.oopspecindex,
+            write_descrs_arrays=self.write_descrs_arrays
+        )
+
 
 class TestHeapCache(object):
     def test_known_class_box(self):
@@ -364,13 +371,13 @@
         # Just need the destination box for this call
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY),
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
             [None, None, box2, None, None]
         )
         assert h.getarrayitem(box1, index1, descr1) is box2
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY),
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
             [None, None, box3, None, None]
         )
         assert h.getarrayitem(box1, index1, descr1) is None
@@ -379,11 +386,24 @@
         assert h.getarrayitem(box4, index1, descr1) is box2
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY),
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
             [None, None, box2, None, None]
         )
         assert h.getarrayitem(box4, index1, descr1) is None
 
+    def test_ll_arraycopy_differing_descrs(self):
+        h = HeapCache()
+        h.setarrayitem(box1, index1, box2, descr1)
+        assert h.getarrayitem(box1, index1, descr1) is box2
+        h.new_array(box2, lengthbox2)
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr2]),
+            [None, None, box2, None, None]
+        )
+        assert h.getarrayitem(box1, index1, descr1) is box2
+
+
     def test_unescaped(self):
         h = HeapCache()
         assert not h.is_unescaped(box1)
diff --git a/rpython/memory/support.py b/rpython/memory/support.py
--- a/rpython/memory/support.py
+++ b/rpython/memory/support.py
@@ -121,13 +121,15 @@
                 cur = next
             free_non_gc_object(self)
 
-        def _length_estimate(self):
+        def length(self):
             chunk = self.chunk
+            result = 0
             count = self.used_in_last_chunk
             while chunk:
+                result += count
                 chunk = chunk.next
-                count += chunk_size
-            return count
+                count = chunk_size
+            return result
 
         def foreach(self, callback, arg):
             """Invoke 'callback(address, arg)' for all addresses in the stack.
@@ -144,7 +146,7 @@
         foreach._annspecialcase_ = 'specialize:arg(1)'
 
         def stack2dict(self):
-            result = AddressDict(self._length_estimate())
+            result = AddressDict(self.length())
             self.foreach(_add_in_dict, result)
             return result
 
diff --git a/rpython/memory/test/test_support.py b/rpython/memory/test/test_support.py
--- a/rpython/memory/test/test_support.py
+++ b/rpython/memory/test/test_support.py
@@ -94,6 +94,18 @@
             assert a == addrs[i]
         assert not ll.non_empty()
 
+    def test_length(self):
+        AddressStack = get_address_stack(10)
+        ll = AddressStack()
+        a = raw_malloc(llmemory.sizeof(lltype.Signed))
+        for i in range(42):
+            assert ll.length() == i
+            ll.append(a)
+        for i in range(42-1, -1, -1):
+            b = ll.pop()
+            assert b == a
+            assert ll.length() == i
+
 
 class TestAddressDeque:
     def test_big_access(self):
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -728,10 +728,16 @@
             if c.numdigits() == 1 and c._digits[0] == ONEDIGIT:
                 return NULLRBIGINT
 
-            # if base < 0:
-            #     base = base % modulus
-            # Having the base positive just makes things easier.
-            if a.sign < 0:
+            # Reduce base by modulus in some cases:
+            # 1. If base < 0.  Forcing the base non-neg makes things easier.
+            # 2. If base is obviously larger than the modulus.  The "small
+            #    exponent" case later can multiply directly by base repeatedly,
+            #    while the "large exponent" case multiplies directly by base 31
+            #    times.  It can be unboundedly faster to multiply by
+            #    base % modulus instead.
+            # We could _always_ do this reduction, but mod() isn't cheap,
+            # so we only do it when it buys something.
+            if a.sign < 0 or a.numdigits() > c.numdigits():
                 a = a.mod(c)
 
         elif b.sign == 0:
diff --git a/rpython/rlib/rpath.py b/rpython/rlib/rpath.py
--- a/rpython/rlib/rpath.py
+++ b/rpython/rlib/rpath.py
@@ -23,3 +23,103 @@
             return path
 else:
     raise ImportError('Unsupported os: %s' % os.name)
+
+
+def dirname(p):
+    """Returns the directory component of a pathname"""
+    i = p.rfind('/') + 1
+    assert i >= 0
+    head = p[:i]
+    if head and head != '/' * len(head):
+        head = head.rstrip('/')
+    return head
+
+
+def basename(p):
+    """Returns the final component of a pathname"""
+    i = p.rfind('/') + 1
+    assert i >= 0
+    return p[i:]
+
+
+def split(p):
+    """Split a pathname.  Returns tuple "(head, tail)" where "tail" is
+    everything after the final slash.  Either part may be empty."""
+    i = p.rfind('/') + 1
+    assert i >= 0
+    head, tail = p[:i], p[i:]
+    if head and head != '/' * len(head):
+        head = head.rstrip('/')
+    return head, tail
+
+
+def exists(path):
+    """Test whether a path exists.  Returns False for broken symbolic links"""
+    try:
+        assert path is not None
+        os.stat(path)
+    except os.error:
+        return False
+    return True
+
+
+import os
+from os.path import isabs, islink, abspath, normpath
+
+def join(a, p):
+    """Join two or more pathname components, inserting '/' as needed.
+    If any component is an absolute path, all previous path components
+    will be discarded.  An empty last part will result in a path that
+    ends with a separator."""
+    path = a
+    for b in p:
+        if b.startswith('/'):
+            path = b
+        elif path == '' or path.endswith('/'):
+            path +=  b
+        else:
+            path += '/' + b
+    return path
+
+def realpath(filename):
+    """Return the canonical path of the specified filename, eliminating any
+symbolic links encountered in the path."""
+    if isabs(filename):
+        bits = ['/'] + filename.split('/')[1:]
+    else:
+        bits = [''] + filename.split('/')
+
+    for i in range(2, len(bits)+1):
+        component = join(bits[0], bits[1:i])
+        # Resolve symbolic links.
+        if islink(component):
+            resolved = _resolve_link(component)
+            if resolved is None:
+                # Infinite loop -- return original component + rest of the path
+                return abspath(join(component, bits[i:]))
+            else:
+                newpath = join(resolved, bits[i:])
+                return realpath(newpath)
+
+    return abspath(filename)
+
+
+def _resolve_link(path):
+    """Internal helper function.  Takes a path and follows symlinks
+    until we either arrive at something that isn't a symlink, or
+    encounter a path we've seen before (meaning that there's a loop).
+    """
+    paths_seen = {}
+    while islink(path):
+        if path in paths_seen:
+            # Already seen this path, so we must have a symlink loop
+            return None
+        paths_seen[path] = None
+        # Resolve where the link points to
+        resolved = os.readlink(path)
+        if not isabs(resolved):
+            dir = dirname(path)
+            path = normpath(join(dir, [resolved]))
+        else:
+            path = normpath(resolved)
+    return path
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -117,6 +117,8 @@
     """ Container for low-level implementation
     of a lock object
     """
+    _immutable_fields_ = ["_lock"]
+
     def __init__(self, ll_lock):
         self._lock = ll_lock
 
diff --git a/rpython/rlib/test/test_rstacklet.py b/rpython/rlib/test/test_rstacklet.py
--- a/rpython/rlib/test/test_rstacklet.py
+++ b/rpython/rlib/test/test_rstacklet.py
@@ -1,5 +1,6 @@
 import gc, sys
 import py
+import platform
 from rpython.rtyper.tool.rffi_platform import CompilationError
 try:
     from rpython.rlib import rstacklet
@@ -332,6 +333,10 @@
     gc = 'minimark'
     gcrootfinder = 'asmgcc'
 
+    @py.test.mark.skipif("sys.platform != 'linux2' or platform.machine().startswith('arm')")
+    def test_demo1(self):
+        BaseTestStacklet.test_demo1(self)
+
 class TestStackletShadowStack(BaseTestStacklet):
     gc = 'minimark'
     gcrootfinder = 'shadowstack'
diff --git a/rpython/translator/c/src/stacklet/switch_arm_gcc.h b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
--- a/rpython/translator/c/src/stacklet/switch_arm_gcc.h
+++ b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
@@ -1,3 +1,10 @@
+#if __ARM_ARCH__ >= 5
+# define call_reg(x) "blx " #x "\n"
+#elif defined (__ARM_ARCH_4T__)
+# define call_reg(x) "mov lr, pc ; bx " #x "\n"
+#else
+# define call_reg(x) "mov lr, pc ; mov pc, " #x "\n"
+#endif
 
 static void __attribute__((optimize("O3"))) *slp_switch(void *(*save_state)(void*, void*),
                         void *(*restore_state)(void*, void*),
@@ -11,7 +18,7 @@
     "mov r5, %[extra]\n"
     "mov r0, sp\n"        	/* arg 1: current (old) stack pointer */
     "mov r1, r5\n"        	/* arg 2: extra                       */
-    "blx r3\n"				/* call save_state()                  */
+    call_reg(r3)		/* call save_state()                  */
 
     /* skip the rest if the return value is null */
     "cmp r0, #0\n"
@@ -23,7 +30,7 @@
 	stack is not restored yet.  It contains only garbage here. */
     "mov r1, r5\n"       	/* arg 2: extra                       */
    	 						/* arg 1: current (new) stack pointer is already in r0*/
-    "blx r4\n"           	/* call restore_state()               */
+    call_reg(r4)		/* call restore_state()               */
 
     /* The stack's content is now restored. */
     "zero:\n"

From noreply at buildbot.pypy.org  Tue Oct  8 07:16:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 07:16:32 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add asserts, tweak debug prints
Message-ID: <20131008051633.00AFD1C0223@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r535:9149deb7e746
Date: 2013-10-08 07:16 +0200
http://bitbucket.org/pypy/stmgc/changeset/9149deb7e746/

Log:	Add asserts, tweak debug prints

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -6,9 +6,9 @@
  */
 #include "stmimpl.h"
 
-char tmp_buf[128];
 char* stm_dbg_get_hdr_str(gcptr obj)
 {
+    static char tmp_buf[128];
     char *cur;
     char *flags[] = GC_FLAG_NAMES;
     int i;
@@ -43,8 +43,10 @@
         struct tx_descriptor *d = stm_tx_head;
         while (d && d->public_descriptor != pd)
             d = d->tx_next;
-        if (!d)
+        if (!d) {
+            fprintf(stderr, "\n");
             continue;
+        }
 
         fprintf(stderr, "((struct tx_descriptor *)\033[%dm%p\033[0m)\n"
                 "pthread_self = 0x%lx\n\n", d->tcolor, d, (long)d->pthreadid);
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -512,6 +512,9 @@
     }
     else {
         d->nursery_cleared = NC_REGULAR;
+#if defined(_GC_DEBUG)
+        memset(d->nursery_current, 0xEE, d->nursery_end - d->nursery_current);
+#endif
     }
 
     /* if in debugging mode, we allocate a different nursery and make
diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -193,6 +193,7 @@
             if (L->h_tid & GCFLAG_HAS_ID) {
                 /* use id-copy for us */
                 O = (gcptr)L->h_original;
+                assert(O != L);
                 L->h_tid &= ~GCFLAG_HAS_ID;
                 stm_copy_to_old_id_copy(L, O);
                 O->h_original = 0;
@@ -200,6 +201,7 @@
                 /* Copy the object out of the other thread's nursery, 
                    if needed */
                 O = stmgc_duplicate_old(L);
+                assert(O != L);
 
                 /* young and without original? */
                 if (!(L->h_original))

From noreply at buildbot.pypy.org  Tue Oct  8 07:17:59 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 07:17:59 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: import stmgc/9149deb7e746
Message-ID: <20131008051759.276C31C0223@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r67187:ef344f3f8725
Date: 2013-10-08 07:16 +0200
http://bitbucket.org/pypy/pypy/changeset/ef344f3f8725/

Log:	import stmgc/9149deb7e746

diff --git a/rpython/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c
--- a/rpython/translator/stm/src_stm/et.c
+++ b/rpython/translator/stm/src_stm/et.c
@@ -7,9 +7,9 @@
  */
 #include "stmimpl.h"
 
-char tmp_buf[128];
 char* stm_dbg_get_hdr_str(gcptr obj)
 {
+    static char tmp_buf[128];
     char *cur;
     char *flags[] = GC_FLAG_NAMES;
     int i;
@@ -44,8 +44,10 @@
         struct tx_descriptor *d = stm_tx_head;
         while (d && d->public_descriptor != pd)
             d = d->tx_next;
-        if (!d)
+        if (!d) {
+            fprintf(stderr, "\n");
             continue;
+        }
 
         fprintf(stderr, "((struct tx_descriptor *)\033[%dm%p\033[0m)\n"
                 "pthread_self = 0x%lx\n\n", d->tcolor, d, (long)d->pthreadid);
diff --git a/rpython/translator/stm/src_stm/nursery.c b/rpython/translator/stm/src_stm/nursery.c
--- a/rpython/translator/stm/src_stm/nursery.c
+++ b/rpython/translator/stm/src_stm/nursery.c
@@ -513,6 +513,9 @@
     }
     else {
         d->nursery_cleared = NC_REGULAR;
+#if defined(_GC_DEBUG)
+        memset(d->nursery_current, 0xEE, d->nursery_end - d->nursery_current);
+#endif
     }
 
     /* if in debugging mode, we allocate a different nursery and make
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-d78107007cab
+9149deb7e746
diff --git a/rpython/translator/stm/src_stm/steal.c b/rpython/translator/stm/src_stm/steal.c
--- a/rpython/translator/stm/src_stm/steal.c
+++ b/rpython/translator/stm/src_stm/steal.c
@@ -194,6 +194,7 @@
             if (L->h_tid & GCFLAG_HAS_ID) {
                 /* use id-copy for us */
                 O = (gcptr)L->h_original;
+                assert(O != L);
                 L->h_tid &= ~GCFLAG_HAS_ID;
                 stm_copy_to_old_id_copy(L, O);
                 O->h_original = 0;
@@ -201,6 +202,7 @@
                 /* Copy the object out of the other thread's nursery, 
                    if needed */
                 O = stmgc_duplicate_old(L);
+                assert(O != L);
 
                 /* young and without original? */
                 if (!(L->h_original))

From noreply at buildbot.pypy.org  Tue Oct  8 07:21:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 07:21:23 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Silence a warning
Message-ID: <20131008052123.18ABE1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stmgc-c4
Changeset: r67188:7eff07c1f5e1
Date: 2013-10-08 07:20 +0200
http://bitbucket.org/pypy/pypy/changeset/7eff07c1f5e1/

Log:	Silence a warning

diff --git a/rpython/translator/c/src/dtoa.c b/rpython/translator/c/src/dtoa.c
--- a/rpython/translator/c/src/dtoa.c
+++ b/rpython/translator/c/src/dtoa.c
@@ -2969,7 +2969,7 @@
     result = __Py_dg_dtoa(dd, mode, ndigits, decpt, sign, rve);
     _PyPy_SET_53BIT_PRECISION_END;
 #ifdef RPY_STM
-    stm_call_on_abort(result, _PyPy_dg_freedtoa);
+    stm_call_on_abort(result, (void(*)(void *))_PyPy_dg_freedtoa);
 #endif
     return result;
 }

From noreply at buildbot.pypy.org  Tue Oct  8 09:38:48 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 09:38:48 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: in-progress
Message-ID: <20131008073848.A53811C3046@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67189:d108922be83a
Date: 2013-10-08 09:38 +0200
http://bitbucket.org/pypy/pypy/changeset/d108922be83a/

Log:	in-progress

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -325,8 +325,6 @@
             ArenaCollectionClass = minimarkpage.ArenaCollection
         self.ac = ArenaCollectionClass(arena_size, page_size,
                                        small_request_threshold)
-        self.ac_alternate = ArenaCollectionClass(arena_size, page_size,
-                                       small_request_threshold)
         #
         # Used by minor collection: a list of (mostly non-young) objects that
         # (may) contain a pointer to a young object.  Populated by
@@ -1015,8 +1013,7 @@
         """Return the total memory used, not counting any object in the
         nursery: only objects in the ArenaCollection or raw-malloced.
         """
-        return self.ac.total_memory_used + self.ac_alternate.total_memory_used \
-                    + self.rawmalloced_total_size
+        return self.ac.total_memory_used + self.rawmalloced_total_size
 
 
     def card_marking_words_for_length(self, length):
@@ -1689,11 +1686,6 @@
         # Copy it.  Note that references to other objects in the
         # nursery are kept unchanged in this step.
         llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
-        # if the current state is sweeping or later,
-        # then all new objects surviving
-        # minor collection should be marked as visited
-        if self.gc_state >= STATE_SWEEPING_RAWMALLOC:
-            self.header(newhdr + size_gc_header).tid |= GCFLAG_VISITED
         #
         # Set the old object's tid to -42 (containing all flags) and
         # replace the old object's content with the target address.
@@ -1718,8 +1710,10 @@
         if self.has_gcptr(typeid):
             # we only have to do it if we have any gcptrs
             self.old_objects_pointing_to_young.append(newobj)
-
-
+        #
+        # If we are in STATE_MARKING, then the new object must be made gray.
+        if self.gc_state == STATE_MARKING:
+            self.write_to_visited_object_backward(newobj)
 
     _trace_drag_out._always_inline_ = True
 
@@ -1838,7 +1832,7 @@
             debug_print("number of objects to mark",
                         self.objects_to_trace.length())
 
-            estimate = 2000
+            estimate = self.nursery_size    # XXX
             self.visit_all_objects_step(estimate)
 
             # XXX A simplifying assumption that should be checked,
@@ -1860,6 +1854,7 @@
                 #objects_to_trace processed fully, can move on to sweeping
                 self.gc_state = STATE_SWEEPING_RAWMALLOC
                 #prepare for the next state
+                self.ac.mass_free_prepare()
                 self.start_free_rawmalloc_objects()
             #END MARKING
         elif self.gc_state == STATE_SWEEPING_RAWMALLOC:
@@ -1876,40 +1871,42 @@
             # Ask the ArenaCollection to visit all objects.  Free the ones
             # that have not been visited above, and reset GCFLAG_VISITED on
             # the others.
-            self.ac.mass_free(self._free_if_unvisited)
-            self.num_major_collects += 1
-            #
-            # We also need to reset the GCFLAG_VISITED on prebuilt GC objects.
-            self.prebuilt_root_objects.foreach(self._reset_gcflag_visited, None)
-            #
-            # Set the threshold for the next major collection to be when we
-            # have allocated 'major_collection_threshold' times more than
-            # we currently have -- but no more than 'max_delta' more than
-            # we currently have.
-            total_memory_used = float(self.get_total_memory_used())
-            bounded = self.set_major_threshold_from(
-                min(total_memory_used * self.major_collection_threshold,
-                    total_memory_used + self.max_delta),
-                reserving_size)
-            #
-            # Max heap size: gives an upper bound on the threshold.  If we
-            # already have at least this much allocated, raise MemoryError.
-            if bounded and (float(self.get_total_memory_used()) + reserving_size >=
-                            self.next_major_collection_initial):
+            max_pages = 3 * (self.nursery_size // self.ac.page_size)  # XXX
+            if self.ac.mass_free_incremental(self._free_if_unvisited,
+                                             max_pages):
+                self.num_major_collects += 1
                 #
-                # First raise MemoryError, giving the program a chance to
-                # quit cleanly.  It might still allocate in the nursery,
-                # which might eventually be emptied, triggering another
-                # major collect and (possibly) reaching here again with an
-                # even higher memory consumption.  To prevent it, if it's
-                # the second time we are here, then abort the program.
-                if self.max_heap_size_already_raised:
-                    llop.debug_fatalerror(lltype.Void,
-                                          "Using too much memory, aborting")
-                self.max_heap_size_already_raised = True
-                raise MemoryError
+                # We also need to reset the GCFLAG_VISITED on prebuilt GC objects.
+                self.prebuilt_root_objects.foreach(self._reset_gcflag_visited, None)
+                #
+                # Set the threshold for the next major collection to be when we
+                # have allocated 'major_collection_threshold' times more than
+                # we currently have -- but no more than 'max_delta' more than
+                # we currently have.
+                total_memory_used = float(self.get_total_memory_used())
+                bounded = self.set_major_threshold_from(
+                    min(total_memory_used * self.major_collection_threshold,
+                        total_memory_used + self.max_delta),
+                    reserving_size)
+                #
+                # Max heap size: gives an upper bound on the threshold.  If we
+                # already have at least this much allocated, raise MemoryError.
+                if bounded and (float(self.get_total_memory_used()) + reserving_size >=
+                                self.next_major_collection_initial):
+                    #
+                    # First raise MemoryError, giving the program a chance to
+                    # quit cleanly.  It might still allocate in the nursery,
+                    # which might eventually be emptied, triggering another
+                    # major collect and (possibly) reaching here again with an
+                    # even higher memory consumption.  To prevent it, if it's
+                    # the second time we are here, then abort the program.
+                    if self.max_heap_size_already_raised:
+                        llop.debug_fatalerror(lltype.Void,
+                                              "Using too much memory, aborting")
+                    self.max_heap_size_already_raised = True
+                    raise MemoryError
 
-            self.gc_state = STATE_FINALIZING
+                self.gc_state = STATE_FINALIZING
             # FINALIZING not yet incrementalised
             # but it seems safe to allow mutator to run after sweeping and
             # before finalizers are called. This is because run_finalizers
@@ -1995,6 +1992,7 @@
         while nobjects > 0 and self.raw_malloc_might_sweep.non_empty():
             self.free_rawmalloced_object_if_unvisited(
                                              self.raw_malloc_might_sweep.pop())
+            nobjects -= 1
 
         if not self.raw_malloc_might_sweep.non_empty():
             self.raw_malloc_might_sweep.delete()
diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -280,7 +280,7 @@
         #
         # Maybe we are incrementally collecting, in which case an arena
         # could have more free pages thrown into it than arenas_lists[]
-        # account for.  Rehash and retry.
+        # accounts for.  Rehash and retry.
         self._rehash_arenas_lists()
         if self._pick_next_arena():
             return
diff --git a/rpython/memory/gc/minimarktest.py b/rpython/memory/gc/minimarktest.py
--- a/rpython/memory/gc/minimarktest.py
+++ b/rpython/memory/gc/minimarktest.py
@@ -1,3 +1,4 @@
+import sys
 from rpython.rtyper.lltypesystem import llarena
 from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage
 from rpython.rlib.debug import ll_assert
@@ -32,13 +33,26 @@
         self.total_memory_used += nsize
         return result
 
-    def mass_free(self, ok_to_free_func):
-        objs = self.all_objects
+    def mass_free_prepare(self):
+        self.old_all_objects = self.all_objects
         self.all_objects = []
         self.total_memory_used = 0
-        for rawobj, nsize in objs:
+
+    def mass_free_incremental(self, ok_to_free_func, max_pages):
+        old = self.old_all_objects
+        while old:
+            rawobj, nsize = old.pop()
             if ok_to_free_func(rawobj):
                 llarena.arena_free(rawobj)
             else:
                 self.all_objects.append((rawobj, nsize))
                 self.total_memory_used += nsize
+            max_pages -= 0.1
+            if max_pages <= 0:
+                return False
+        return True
+
+    def mass_free(self, ok_to_free_func):
+        self.mass_free_prepare()
+        res = self.mass_free_incremental(ok_to_free_func, sys.maxint)
+        assert res

From noreply at buildbot.pypy.org  Tue Oct  8 09:58:39 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 09:58:39 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Kill major_collection().
Message-ID: <20131008075839.DAF231C0161@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67190:8c4c2c336821
Date: 2013-10-08 09:50 +0200
http://bitbucket.org/pypy/pypy/changeset/8c4c2c336821/

Log:	Kill major_collection().

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -669,10 +669,17 @@
 
 
     def collect(self, gen=1):
-        """Do a minor (gen=0) or major (gen>0) collection."""
+        """Do a minor (gen=0) or full major (gen>0) collection."""
         self.minor_collection()
         if gen > 0:
-            self.major_collection()
+            #
+            # First, finish the current major gc, if there is one in progress.
+            # This is a no-op if the gc_state is already STATE_SCANNING.
+            self.gc_step_until(STATE_SCANNING)
+            #
+            # Then do a complete collection again.
+            self.gc_step_until(STATE_MARKING)
+            self.gc_step_until(STATE_SCANNING)
 
     def move_nursery_top(self, totalsize):
         size = self.nursery_cleanup
@@ -1796,11 +1803,13 @@
             old.append(new.pop())
         new.delete()
 
-    def debug_gc_step_until(self,state):
+    def gc_step_until(self,state):
         while self.gc_state != state:
             self.minor_collection()
             self.major_collection_step()
 
+    debug_gc_step_until = gc_step_until   # xxx
+
     def debug_gc_step(self, n=1):
         while n > 0:
             self.minor_collection()
@@ -1927,19 +1936,6 @@
 
         debug_stop("gc-collect-step")
 
-    def major_collection(self, reserving_size=0):
-        # For now keep things compatible with the existing GC
-        # and do all steps in a loop
-
-        # We start in scanning state
-        ll_assert(self.gc_state == STATE_SCANNING,
-                    "Scan start state incorrect")
-        self.major_collection_step(reserving_size)
-        ll_assert(self.gc_state == STATE_MARKING, "initial scan did not complete")
-
-        while self.gc_state != STATE_SCANNING:
-            self.major_collection_step(reserving_size)
-
     def _free_if_unvisited(self, hdr):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         obj = hdr + size_gc_header

From noreply at buildbot.pypy.org  Tue Oct  8 09:58:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 09:58:41 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Added a TODO
Message-ID: <20131008075841.0B9C91C0161@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67191:dde7f7507abb
Date: 2013-10-08 09:57 +0200
http://bitbucket.org/pypy/pypy/changeset/dde7f7507abb/

Log:	Added a TODO

diff --git a/TODO b/TODO
new file mode 100644
--- /dev/null
+++ b/TODO
@@ -0,0 +1,13 @@
+
+
+* fix increments in major_collection_step() in the phases
+  STATE_MARKING, STATE_SWEEPING_RAWMALLOC, STATE_SWEEPING_ARENA,
+  and probably STATE_FINALIZING
+
+* 'next_major_collection_*' need to be tweaked
+
+* check the 'reserving_size' argument here and there
+
+* maybe make incremental: dealing with old_objects_with_weakrefs
+  and old_objects_with_light_finalizers and
+  deal_with_objects_with_finalizers()

From noreply at buildbot.pypy.org  Tue Oct  8 10:03:58 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 10:03:58 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Disable this logic
Message-ID: <20131008080358.031E71C02A3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67192:04f75ade7282
Date: 2013-10-08 10:01 +0200
http://bitbucket.org/pypy/pypy/changeset/04f75ade7282/

Log:	Disable this logic

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -11,3 +11,6 @@
 * maybe make incremental: dealing with old_objects_with_weakrefs
   and old_objects_with_light_finalizers and
   deal_with_objects_with_finalizers()
+
+* REDO external_malloc(): if somebody calls this function a lot, we must
+  eventually force a full collection.
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -766,10 +766,11 @@
         #
         # If somebody calls this function a lot, we must eventually
         # force a full collection.
-        if (float(self.get_total_memory_used()) + raw_malloc_usage(totalsize) >
-                self.next_major_collection_threshold):
-            self.minor_collection()
-            self.major_collection(raw_malloc_usage(totalsize))
+        # XXX REDO
+##        if (float(self.get_total_memory_used()) + raw_malloc_usage(totalsize) >
+##                self.next_major_collection_threshold):
+##            self.minor_collection()
+##            self.major_collection(raw_malloc_usage(totalsize))
         #
         # Check if the object would fit in the ArenaCollection.
         if raw_malloc_usage(totalsize) <= self.small_request_threshold:

From noreply at buildbot.pypy.org  Tue Oct  8 11:26:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 11:26:32 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Fix fix
Message-ID: <20131008092632.CF5A51C02A3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67193:7aceddb4abfb
Date: 2013-10-08 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/7aceddb4abfb/

Log:	Fix fix

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -791,6 +791,7 @@
             # is for large objects, bigger than the 'large_objects' threshold,
             # which are raw-malloced but still young.
             extra_flags = GCFLAG_TRACK_YOUNG_PTRS
+            can_make_young = False
             #
         else:
             # No, so proceed to allocate it externally with raw_malloc().
@@ -862,7 +863,13 @@
         if self.is_varsize(typeid):
             offset_to_length = self.varsize_offset_to_length(typeid)
             (result + size_gc_header + offset_to_length).signed[0] = length
-        return result + size_gc_header
+        newobj = result + size_gc_header
+        #
+        # If we are in STATE_MARKING, then the new object must be made gray.
+        if not can_make_young and self.gc_state == STATE_MARKING:
+            self.write_to_visited_object_backward(newobj)
+        #
+        return newobj
 
 
     # ----------
@@ -1789,7 +1796,9 @@
     def _free_young_rawmalloced_obj(self, obj, ignored1, ignored2):
         # If 'obj' has GCFLAG_VISITED, it was seen by _trace_drag_out
         # and survives.  Otherwise, it dies.
-        self.free_rawmalloced_object_if_unvisited(obj)
+        if not self.free_rawmalloced_object_if_unvisited(obj):
+            if self.gc_state == STATE_MARKING:
+                self.write_to_visited_object_backward(obj)
 
     def remove_young_arrays_from_old_objects_pointing_to_young(self):
         old = self.old_objects_pointing_to_young
@@ -1935,6 +1944,7 @@
         else:
             pass #XXX which exception to raise here. Should be unreachable.
 
+        debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
         debug_stop("gc-collect-step")
 
     def _free_if_unvisited(self, hdr):
@@ -1956,6 +1966,7 @@
             self.header(obj).tid |= GCFLAG_OLD
             self.header(obj).tid &= ~(GCFLAG_VISITED|GCFLAG_GRAY)   # survives
             self.old_rawmalloced_objects.append(obj)
+            return False
         else:
             size_gc_header = self.gcheaderbuilder.size_gc_header
             totalsize = size_gc_header + self.get_size(obj)
@@ -1978,6 +1989,7 @@
             #
             llarena.arena_free(arena)
             self.rawmalloced_total_size -= r_uint(allocsize)
+            return True
 
     def start_free_rawmalloc_objects(self):
         self.raw_malloc_might_sweep = self.old_rawmalloced_objects
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -202,9 +202,17 @@
                 assert self.stackroots[index][index2].x == value
         x = 0
         for i in range(40):
-            self.stackroots.append(self.malloc(VAR, i))
+            assert 'DEAD' not in repr(self.stackroots)
+            a = self.malloc(VAR, i)
+            assert 'DEAD' not in repr(a)
+            self.stackroots.append(a)
+            print 'ADDED TO STACKROOTS:', llmemory.cast_adr_to_int(
+                llmemory.cast_ptr_to_adr(a))
+            assert 'DEAD' not in repr(self.stackroots)
             for j in range(5):
+                assert 'DEAD' not in repr(self.stackroots)
                 p = self.malloc(S)
+                assert 'DEAD' not in repr(self.stackroots)
                 p.x = x
                 index = x % len(self.stackroots)
                 if index > 0:
@@ -685,6 +693,7 @@
     # Test trying to be a bit comprehensive about
     # states and types of objects
     def test_allocate_states(self):
+        py.test.skip("broken test for now")
         from rpython.memory.gc import incminimark
         largeobj_size =  self.gc.nonlarge_max + 1
 
@@ -765,5 +774,5 @@
         for obj in unreachable:
             assert py.test.raises(RuntimeError,"obj.x")
 
-class TestIncrementalMiniMarkGCFull(TestMiniMarkGCFull):
+class TestIncrementalMiniMarkGCFull(DirectGCTest):
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass

From noreply at buildbot.pypy.org  Tue Oct  8 11:29:33 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Tue,  8 Oct 2013 11:29:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: add a note to tests that use dns results
Message-ID: <20131008092933.B5A641C02A3@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r67194:5da47a126c0b
Date: 2013-10-08 04:27 -0500
http://bitbucket.org/pypy/pypy/changeset/5da47a126c0b/

Log:	add a note to tests that use dns results

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -400,6 +400,8 @@
     def test_socket_connect_ex(self):
         import _socket
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+        # The following might fail if the DNS redirects failed requests to a
+        # catch-all address (i.e. opendns).
         # Make sure we get an app-level error, not an interp one.
         raises(_socket.gaierror, s.connect_ex, ("wrong.invalid", 80))
         s.close()
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -301,6 +301,8 @@
             addr.get_port() == 80):
             found = True
     assert found, lst
+    # The following might fail if the DNS redirects failed requests to a
+    # catch-all address (i.e. opendns).
     e = py.test.raises(GAIError, getaddrinfo, 'www.very-invalidaddress.com', None)
     assert isinstance(e.value.get_msg(), str)
 

From noreply at buildbot.pypy.org  Tue Oct  8 11:30:50 2013
From: noreply at buildbot.pypy.org (zyv)
Date: Tue,  8 Oct 2013 11:30:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: CPython C-API compat:
 PyErr_BadArgument() is of type int and always returns zero
Message-ID: <20131008093050.BF7C61C02A3@cobra.cs.uni-duesseldorf.de>

Author: Yury V. Zaytsev <yury at shurup.com>
Branch: 
Changeset: r67195:732ce43952a5
Date: 2013-10-08 10:12 +0200
http://bitbucket.org/pypy/pypy/changeset/732ce43952a5/

Log:	CPython C-API compat: PyErr_BadArgument() is of type int and always
	returns zero

diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -103,11 +103,13 @@
     exc_p[0] = make_ref(space, operr.w_type)
     val_p[0] = make_ref(space, operr.get_w_value(space))
 
- at cpython_api([], lltype.Void)
+ at cpython_api([], rffi.INT_real, error=0)
 def PyErr_BadArgument(space):
     """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where
     message indicates that a built-in operation was invoked with an illegal
-    argument.  It is mostly for internal use."""
+    argument.  It is mostly for internal use. In CPython this function always
+    raises an exception and returns 0 in all cases, hence the (ab)use of the
+    error indicator."""
     raise OperationError(space.w_TypeError,
             space.wrap("bad argument type for built-in operation"))
 
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -70,9 +70,10 @@
         api.PyErr_Clear()
 
     def test_BadArgument(self, space, api):
-        api.PyErr_BadArgument()
+        ret = api.PyErr_BadArgument()
         state = space.fromcache(State)
         assert space.eq_w(state.operror.w_type, space.w_TypeError)
+        assert ret == 0
         api.PyErr_Clear()
 
     def test_Warning(self, space, api, capfd):

From noreply at buildbot.pypy.org  Tue Oct  8 11:30:52 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 11:30:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged in zyv/pypy (pull request #193)
Message-ID: <20131008093052.D353B1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67196:4852d98410ec
Date: 2013-10-08 11:30 +0200
http://bitbucket.org/pypy/pypy/changeset/4852d98410ec/

Log:	Merged in zyv/pypy (pull request #193)

	CPython C-API compat: PyErr_BadArgument() is of type int and always
	returns zero

diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -103,11 +103,13 @@
     exc_p[0] = make_ref(space, operr.w_type)
     val_p[0] = make_ref(space, operr.get_w_value(space))
 
- at cpython_api([], lltype.Void)
+ at cpython_api([], rffi.INT_real, error=0)
 def PyErr_BadArgument(space):
     """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where
     message indicates that a built-in operation was invoked with an illegal
-    argument.  It is mostly for internal use."""
+    argument.  It is mostly for internal use. In CPython this function always
+    raises an exception and returns 0 in all cases, hence the (ab)use of the
+    error indicator."""
     raise OperationError(space.w_TypeError,
             space.wrap("bad argument type for built-in operation"))
 
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -70,9 +70,10 @@
         api.PyErr_Clear()
 
     def test_BadArgument(self, space, api):
-        api.PyErr_BadArgument()
+        ret = api.PyErr_BadArgument()
         state = space.fromcache(State)
         assert space.eq_w(state.operror.w_type, space.w_TypeError)
+        assert ret == 0
         api.PyErr_Clear()
 
     def test_Warning(self, space, api, capfd):

From noreply at buildbot.pypy.org  Tue Oct  8 11:52:54 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 11:52:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: probably a fix for translating the gc
 without card marking
Message-ID: <20131008095254.D87A11C026D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67197:bbc29c671a32
Date: 2013-10-08 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/bbc29c671a32/

Log:	probably a fix for translating the gc without card marking

diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1201,6 +1201,7 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+            assert self.card_page_indices > 0
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.

From noreply at buildbot.pypy.org  Tue Oct  8 11:54:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 11:54:49 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Translates and generally pass
	tests
Message-ID: <20131008095449.AD48C1C026D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67198:0fe88e84d8bc
Date: 2013-10-08 11:54 +0200
http://bitbucket.org/pypy/pypy/changeset/0fe88e84d8bc/

Log:	Translates and generally pass tests

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -14,3 +14,6 @@
 
 * REDO external_malloc(): if somebody calls this function a lot, we must
   eventually force a full collection.
+
+* REDO card marking, starting with "card_page_indices": 128 in
+  TRANSLATION_PARAMS
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -256,7 +256,7 @@
         # value of 128 means that card pages are 512 bytes (1024 on 64-bits)
         # in regular arrays of pointers; more in arrays whose items are
         # larger.  A value of 0 disables card marking.
-        "card_page_indices": 128,
+        "card_page_indices": 0,   # XXX was 128,
 
         # Objects whose total size is at least 'large_object' bytes are
         # allocated out of the nursery immediately, as old objects.  The
@@ -1410,6 +1410,7 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+            assert self.card_page_indices > 0
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1468,72 +1468,9 @@
         res = self.run("nongc_opaque_attached_to_gc")
         assert res == 0
 
-class TestIncrementalMiniMarkGC(TestSemiSpaceGC):
+
+class TestIncrementalMiniMarkGC(TestMiniMarkGC):
     gcpolicy = "incminimark"
-    should_be_moving = True
-    GC_CAN_MALLOC_NONMOVABLE = True
-    GC_CAN_SHRINK_ARRAY = True
-
-    def test_gc_heap_stats(self):
-        py.test.skip("not implemented")
-
-    def define_nongc_attached_to_gc(cls):
-        from rpython.rtyper.lltypesystem import rffi
-        ARRAY = rffi.CArray(rffi.INT)
-        class A:
-            def __init__(self, n):
-                self.buf = lltype.malloc(ARRAY, n, flavor='raw',
-                                         add_memory_pressure=True)
-            def __del__(self):
-                lltype.free(self.buf, flavor='raw')
-        A(6)
-        def f():
-            # allocate a total of ~77GB, but if the automatic gc'ing works,
-            # it should never need more than a few MBs at once
-            am1 = am2 = am3 = None
-            res = 0
-            for i in range(1, 100001):
-                if am3 is not None:
-                    res += rffi.cast(lltype.Signed, am3.buf[0])
-                am3 = am2
-                am2 = am1
-                am1 = A(i * 4)
-                am1.buf[0] = rffi.cast(rffi.INT, i - 50000)
-            return res
-        return f
-
-    def test_nongc_attached_to_gc(self):
-        res = self.run("nongc_attached_to_gc")
-        assert res == -99997
-
-    def define_nongc_opaque_attached_to_gc(cls):
-        from rpython.rlib import rgc, ropenssl
-
-        class A:
-            def __init__(self):
-                self.ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO,
-                    flavor='raw')
-                digest = ropenssl.EVP_get_digestbyname('sha1')
-                ropenssl.EVP_DigestInit(self.ctx, digest)
-                rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64)
-
-            def __del__(self):
-                ropenssl.EVP_MD_CTX_cleanup(self.ctx)
-                lltype.free(self.ctx, flavor='raw')
-        #A() --- can't call it here?? get glibc crashes on tannit64
-        def f():
-            am1 = am2 = am3 = None
-            for i in range(100000):
-                am3 = am2
-                am2 = am1
-                am1 = A()
-            # what can we use for the res?
-            return 0
-        return f
-
-    def test_nongc_opaque_attached_to_gc(self):
-        res = self.run("nongc_opaque_attached_to_gc")
-        assert res == 0
 
 
 # ____________________________________________________________________

From noreply at buildbot.pypy.org  Tue Oct  8 11:56:05 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 11:56:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add the option '--gc=incminimark'
Message-ID: <20131008095605.888671C026D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67199:0dc48acab437
Date: 2013-10-08 11:55 +0200
http://bitbucket.org/pypy/pypy/changeset/0dc48acab437/

Log:	Add the option '--gc=incminimark'

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -50,7 +50,7 @@
     # gc
     ChoiceOption("gc", "Garbage Collection Strategy",
                  ["boehm", "ref", "semispace", "statistics",
-                  "generation", "hybrid", "minimark", "none"],
+                  "generation", "hybrid", "minimark", "incminimark", "none"],
                   "ref", requires={
                      "ref": [("translation.rweakref", False), # XXX
                              ("translation.gctransformer", "ref")],
@@ -63,6 +63,7 @@
                      "boehm": [("translation.continuation", False),  # breaks
                                ("translation.gctransformer", "boehm")],
                      "minimark": [("translation.gctransformer", "framework")],
+                     "incminimark": [("translation.gctransformer", "framework")],
                      },
                   cmdline="--gc"),
     ChoiceOption("gctransformer", "GC transformer that is used - internal",

From noreply at buildbot.pypy.org  Tue Oct  8 11:57:22 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 11:57:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: Backed out changeset 0dc48acab437
Message-ID: <20131008095722.7D8B61C026D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67200:8507ff6bb9c9
Date: 2013-10-08 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/8507ff6bb9c9/

Log:	Backed out changeset 0dc48acab437

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -50,7 +50,7 @@
     # gc
     ChoiceOption("gc", "Garbage Collection Strategy",
                  ["boehm", "ref", "semispace", "statistics",
-                  "generation", "hybrid", "minimark", "incminimark", "none"],
+                  "generation", "hybrid", "minimark", "none"],
                   "ref", requires={
                      "ref": [("translation.rweakref", False), # XXX
                              ("translation.gctransformer", "ref")],
@@ -63,7 +63,6 @@
                      "boehm": [("translation.continuation", False),  # breaks
                                ("translation.gctransformer", "boehm")],
                      "minimark": [("translation.gctransformer", "framework")],
-                     "incminimark": [("translation.gctransformer", "framework")],
                      },
                   cmdline="--gc"),
     ChoiceOption("gctransformer", "GC transformer that is used - internal",

From noreply at buildbot.pypy.org  Tue Oct  8 14:10:10 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 14:10:10 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: JIT progress
Message-ID: <20131008121010.1B7ED1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67201:f3aee47b1820
Date: 2013-10-08 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/f3aee47b1820/

Log:	JIT progress

diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -357,7 +357,8 @@
     def _check_valid_gc(self):
         # we need the hybrid or minimark GC for rgc._make_sure_does_not_move()
         # to work.  'hybrid' could work but isn't tested with the JIT.
-        if self.gcdescr.config.translation.gc not in ('minimark',):
+        if self.gcdescr.config.translation.gc not in ('minimark',
+                                                      'incminimark'):
             raise NotImplementedError("--gc=%s not implemented with the JIT" %
                                       (self.gcdescr.config.translation.gc,))
 
diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
--- a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
+++ b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
@@ -113,6 +113,7 @@
 
 
 class BaseFrameworkTests(object):
+    gc = DEFL_GC
 
     def setup_class(cls):
         funcs = []
@@ -162,7 +163,7 @@
         OLD_DEBUG = GcLLDescr_framework.DEBUG
         try:
             GcLLDescr_framework.DEBUG = True
-            cls.cbuilder = compile(get_entry(allfuncs), DEFL_GC,
+            cls.cbuilder = compile(get_entry(allfuncs), cls.gc,
                                    gcrootfinder=cls.gcrootfinder, jit=True,
                                    thread=True)
         finally:
diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py
--- a/rpython/jit/backend/x86/test/test_zrpy_gc.py
+++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py
@@ -3,3 +3,4 @@
 
 class TestShadowStack(CompileFrameworkTests):
     gcrootfinder = "shadowstack"
+    gc = "incminimark"
diff --git a/rpython/jit/metainterp/gc.py b/rpython/jit/metainterp/gc.py
--- a/rpython/jit/metainterp/gc.py
+++ b/rpython/jit/metainterp/gc.py
@@ -25,6 +25,9 @@
 class GC_minimark(GcDescription):
     malloc_zero_filled = True
 
+class GC_incminimark(GcDescription):
+    malloc_zero_filled = True
+
 
 def get_description(config):
     name = config.translation.gc
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1187,7 +1187,7 @@
     # for the JIT: a minimal description of the write_barrier() method
     # (the JIT assumes it is of the shape
     #  "if addr_struct.int0 & JIT_WB_IF_FLAG: remember_young_pointer()")
-    JIT_WB_IF_FLAG = GCFLAG_TRACK_YOUNG_PTRS
+    JIT_WB_IF_FLAG = GCFLAG_TRACK_YOUNG_PTRS | GCFLAG_VISITED
 
     # for the JIT to generate custom code corresponding to the array
     # write barrier for the simplest case of cards.  If JIT_CARDS_SET
@@ -1214,20 +1214,27 @@
     def write_barrier(self, addr_struct):
         if self.header(addr_struct).tid & (GCFLAG_TRACK_YOUNG_PTRS
                                            | GCFLAG_VISITED):
-            self.write_barrier_slowpath(addr_struct)
-
-    def write_barrier_slowpath(self, addr_struct):
-        if self.header(addr_struct).tid & GCFLAG_TRACK_YOUNG_PTRS:
             self.remember_young_pointer(addr_struct)
-        if self.header(addr_struct).tid & GCFLAG_VISITED:
-            if self.gc_state == STATE_MARKING:
-                self.write_to_visited_object_backward(addr_struct)
-    write_barrier_slowpath._dont_inline_ = True
 
     def write_barrier_from_array(self, addr_array, index):
         if self.header(addr_array).tid & (GCFLAG_TRACK_YOUNG_PTRS |
                                           GCFLAG_VISITED):
-            self.write_barrier_slowpath(addr_array)
+            self.remember_young_pointer(addr_array)
+
+    def write_to_visited_object_backward(self, addr_struct):
+        """Call during the marking phase only, when writing into an object
+        that is 'black' in terms of the classical tri-color GC, i.e. that
+        has the GCFLAG_VISITED.  This implements a 'backward' write barrier,
+        i.e. it turns the object back from 'black' to 'gray'.
+        """
+        ll_assert(self.gc_state == STATE_MARKING,"expected MARKING state")
+        # writing a white object into black, make black gray and
+        # readd to objects_to_trace
+        # this is useful for arrays because it stops the writebarrier
+        # from being re-triggered on successive writes
+        self.header(addr_struct).tid &= ~GCFLAG_VISITED
+        self.header(addr_struct).tid |= GCFLAG_GRAY
+        self.objects_to_trace.append(addr_struct)
 
     def _init_writebarrier_logic(self):
         DEBUG = self.DEBUG
@@ -1235,32 +1242,6 @@
         # instead of keeping it as a regular method is to
         # make the code in write_barrier() marginally smaller
         # (which is important because it is inlined *everywhere*).
-
-        # move marking process forward
-        def write_to_visited_object_forward(addr_struct, new_value):
-            ll_assert(self.gc_state == STATE_MARKING,"expected MARKING state")
-            if  self.header(new_value).tid & (GCFLAG_GRAY | GCFLAG_VISITED) == 0:
-                # writing a white object into black, make new object gray and
-                # add to objects_to_trace
-                #
-                self.header(new_value).tid |= GCFLAG_GRAY
-                self.objects_to_trace.append(new_value)
-        write_to_visited_object_forward._dont_inline_ = True
-        self.write_to_visited_object_forward = write_to_visited_object_forward
-
-        # move marking process backward
-        def write_to_visited_object_backward(addr_struct):
-            ll_assert(self.gc_state == STATE_MARKING,"expected MARKING state")
-            # writing a white object into black, make black gray and
-            # readd to objects_to_trace
-            # this is useful for arrays because it stops the writebarrier
-            # from being re-triggered on successive writes
-            self.header(addr_struct).tid &= ~GCFLAG_VISITED
-            self.header(addr_struct).tid |= GCFLAG_GRAY
-            self.objects_to_trace.append(addr_struct)
-        write_to_visited_object_backward._dont_inline_ = True
-        self.write_to_visited_object_backward = write_to_visited_object_backward
-
         def remember_young_pointer(addr_struct):
             # 'addr_struct' is the address of the object in which we write.
             # We know that 'addr_struct' has GCFLAG_TRACK_YOUNG_PTRS so far.
@@ -1270,6 +1251,14 @@
                           self.header(addr_struct).tid & GCFLAG_HAS_CARDS != 0,
                       "young object with GCFLAG_TRACK_YOUNG_PTRS and no cards")
             #
+            # This is the write barrier of incremental GC
+            tid = self.header(addr_struct).tid
+            if tid & GCFLAG_VISITED:
+                if self.gc_state == STATE_MARKING:
+                    self.write_to_visited_object_backward(addr_struct)
+                if tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
+                    return    # done
+            #
             # We need to remove the flag GCFLAG_TRACK_YOUNG_PTRS and add
             # the object to the list 'old_objects_pointing_to_young'.
             # We know that 'addr_struct' cannot be in the nursery,
@@ -1296,17 +1285,6 @@
         remember_young_pointer._dont_inline_ = True
         self.remember_young_pointer = remember_young_pointer
         #
-        def jit_remember_young_pointer(addr_struct):
-            # minimal version of the above, with just one argument,
-            # called by the JIT when GCFLAG_TRACK_YOUNG_PTRS is set
-            self.old_objects_pointing_to_young.append(addr_struct)
-            objhdr = self.header(addr_struct)
-            objhdr.tid &= ~GCFLAG_TRACK_YOUNG_PTRS
-            if objhdr.tid & GCFLAG_NO_HEAP_PTRS:
-                objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS
-                self.prebuilt_root_objects.append(addr_struct)
-        self.jit_remember_young_pointer = jit_remember_young_pointer
-        #
         if self.card_page_indices > 0:
             self._init_writebarrier_with_card_marker()
 
@@ -1366,13 +1344,13 @@
             # called by the JIT when GCFLAG_TRACK_YOUNG_PTRS is set
             # but GCFLAG_CARDS_SET is cleared.  This tries to set
             # GCFLAG_CARDS_SET if possible; otherwise, it falls back
-            # to jit_remember_young_pointer().
+            # to remember_young_pointer().
             objhdr = self.header(addr_array)
             if objhdr.tid & GCFLAG_HAS_CARDS:
                 self.old_objects_with_cards_set.append(addr_array)
                 objhdr.tid |= GCFLAG_CARDS_SET
             else:
-                self.jit_remember_young_pointer(addr_array)
+                self.remember_young_pointer(addr_array)
 
         self.jit_remember_young_pointer_from_array = (
             jit_remember_young_pointer_from_array)
@@ -1410,7 +1388,8 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
-            assert self.card_page_indices > 0
+            if self.card_page_indices == 0:
+                return False     # shouldn't have GCFLAG_HAS_CARDS then...
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -871,8 +871,9 @@
 
     def gct_get_write_barrier_from_array_failing_case(self, hop):
         op = hop.spaceop
-        v = getattr(self, 'write_barrier_from_array_failing_case_ptr',
-                    lltype.nullptr(op.result.concretetype.TO))
+        null = lltype.nullptr(op.result.concretetype.TO)
+        c_null = rmodel.inputconst(op.result.concretetype, null)
+        v = getattr(self, 'write_barrier_from_array_failing_case_ptr', c_null)
         hop.genop("same_as", [v], resultvar=op.result)
 
     def gct_zero_gc_pointers_inside(self, hop):

From noreply at buildbot.pypy.org  Tue Oct  8 14:52:06 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue,  8 Oct 2013 14:52:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix pypy issue 1598
Message-ID: <20131008125206.BC12D1C0161@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r67202:2e8639dfd82e
Date: 2013-10-08 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/2e8639dfd82e/

Log:	Fix pypy issue 1598

diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -319,6 +319,15 @@
         else:
             self.done_func = None
 
+    def are_common_types(self, dtype1, dtype2):
+        if dtype1.is_complex_type() and dtype2.is_complex_type():
+            return True
+        elif not (dtype1.is_complex_type() or dtype2.is_complex_type()) and \
+                (dtype1.is_int_type() and dtype2.is_int_type() or dtype1.is_float_type() and dtype2.is_float_type()) and \
+                not (dtype1.is_bool_type() or dtype2.is_bool_type()):
+            return True
+        return False
+
     @jit.unroll_safe
     def call(self, space, args_w):
         if len(args_w) > 2:
@@ -339,6 +348,12 @@
                  'unsupported operand dtypes %s and %s for "%s"' % \
                  (w_rdtype.get_name(), w_ldtype.get_name(),
                   self.name)))
+
+        if self.are_common_types(w_ldtype, w_rdtype):
+            if not w_lhs.is_scalar() and w_rhs.is_scalar():
+                w_rdtype = w_ldtype
+            elif w_lhs.is_scalar() and not w_rhs.is_scalar():
+                w_ldtype = w_rdtype
         calc_dtype = find_binop_result_dtype(space,
             w_ldtype, w_rdtype,
             int_only=self.int_only,
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2971,6 +2971,11 @@
                         dtype=[('bg', 'i8'), ('fg', 'i8'), ('char', 'S1')])
         assert c[0][0]["char"] == 'a'
 
+    def test_scalar_coercion(self):
+        import numpypy as np
+        a = np.array([1,2,3], dtype=np.int16)
+        assert (a * 2).dtype == np.int16
+
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect and '__pypy__' not in sys.builtin_module_names:

From noreply at buildbot.pypy.org  Tue Oct  8 15:07:48 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 15:07:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: A test for hex(r_uint(..)). The real
 goal is a bit obscure, 
Message-ID: <20131008130748.2AB2F1C0271@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67203:d1174f51656b
Date: 2013-10-08 15:07 +0200
http://bitbucket.org/pypy/pypy/changeset/d1174f51656b/

Log:	A test for hex(r_uint(..)). The real goal is a bit obscure, because
	on top of Python the string contains a final 'L', but should we give
	this behavior in RPython??

diff --git a/rpython/rtyper/test/test_rint.py b/rpython/rtyper/test/test_rint.py
--- a/rpython/rtyper/test/test_rint.py
+++ b/rpython/rtyper/test/test_rint.py
@@ -85,6 +85,14 @@
         res = self.ll_to_string(res)
         assert res == '-0x8' + '0' * (len(res)-4)
 
+    def test_hex_of_uint(self):
+        def dummy(i):
+            return hex(r_uint(i))
+
+        res = self.interpret(dummy, [-5])
+        res = self.ll_to_string(res)
+        assert res == '0x' + 'f' * (len(res)-3) + 'b'
+
     def test_oct_of_int(self):
         def dummy(i):
             return oct(i)

From noreply at buildbot.pypy.org  Tue Oct  8 15:24:40 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 15:24:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix for the error message
Message-ID: <20131008132440.CDFD81C03DF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67204:ae1fe34facf8
Date: 2013-10-08 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/ae1fe34facf8/

Log:	Test and fix for the error message

diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -187,13 +187,15 @@
         if hop.nb_args > 2:
             v_start = hop.inputarg(Signed, arg=2)
             if not hop.args_s[2].nonneg:
-                raise TyperError("str.find() start must be proven non-negative")
+                raise TyperError("str.%s() start must be proven non-negative"
+                                 % (reverse and 'rfind' or 'find',))
         else:
             v_start = hop.inputconst(Signed, 0)
         if hop.nb_args > 3:
             v_end = hop.inputarg(Signed, arg=3)
             if not hop.args_s[3].nonneg:
-                raise TyperError("str.find() end must be proven non-negative")
+                raise TyperError("str.%s() end must be proven non-negative"
+                                 % (reverse and 'rfind' or 'find',))
         else:
             v_end = hop.gendirectcall(self.ll.ll_strlen, v_str)
         hop.exception_cannot_occur()
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -416,6 +416,14 @@
             res = self.interpret(f, [i])
             assert res == expected
 
+    def test_rfind_error_message(self):
+        const = self.const
+        def f(i):
+            return const("abc").rfind(const(''), i)
+        e = py.test.raises(TyperError, self.interpret, f, [-5])
+        assert str(e.value).startswith(
+            'str.rfind() start must be proven non-negative')
+
     def test_find_char(self):
         const = self.const
         def fn(ch):
@@ -1134,4 +1142,4 @@
         array = lltype.malloc(TP, 12, flavor='raw')
         self.interpret(f, [array, 4])
         assert list(array) == list('abc'*4)
-        lltype.free(array, flavor='raw')
\ No newline at end of file
+        lltype.free(array, flavor='raw')

From noreply at buildbot.pypy.org  Tue Oct  8 15:33:45 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 15:33:45 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Increment nobjects here,
 replacing the value "1" with another
Message-ID: <20131008133345.8402D1C02A3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67205:7f52351a3a0a
Date: 2013-10-08 15:33 +0200
http://bitbucket.org/pypy/pypy/changeset/7f52351a3a0a/

Log:	Increment nobjects here, replacing the value "1" with another value
	that is as arbitrary but probably a bit better

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1861,7 +1861,8 @@
             # Walk all rawmalloced objects and free the ones that don't
             # have the GCFLAG_VISITED flag.
             # XXX heuristic here to decide nobjects.
-            if self.free_unvisited_rawmalloc_objects_step(1):
+            nobjects = self.nursery_size // self.ac.page_size  # XXX
+            if self.free_unvisited_rawmalloc_objects_step(nobjects):
                 #malloc objects freed
                 self.gc_state = STATE_SWEEPING_ARENA
 

From noreply at buildbot.pypy.org  Tue Oct  8 16:56:17 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 16:56:17 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: Finish the implementation of
	readline().
Message-ID: <20131008145617.864871C03DF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fileops2
Changeset: r67207:3a0aea85c508
Date: 2013-10-08 16:55 +0200
http://bitbucket.org/pypy/pypy/changeset/3a0aea85c508/

Log:	Finish the implementation of readline().

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -47,7 +47,7 @@
                      rffi.CCHARP)
 
 BASE_BUF_SIZE = 4096
-BASE_LINE_SIZE = 1000
+BASE_LINE_SIZE = 100
 
 def create_file(filename, mode="r", buffering=-1):
     assert buffering == -1
@@ -194,35 +194,48 @@
     def __del__(self):
         self.close()
 
+    def _readline1(self, raw_buf):
+        result = c_fgets(raw_buf, BASE_LINE_SIZE, self.ll_file)
+        if not result:
+            if c_feof(self.ll_file):   # ok
+                return 0
+            errno = c_ferror(self.ll_file)
+            raise OSError(errno, os.strerror(errno))
+        #
+        # Assume that fgets() works as documented, and additionally
+        # never writes beyond the final \0, which the CPython
+        # fileobject.c says appears to be the case everywhere.
+        # The only case where the buffer was not big enough is the
+        # case where the buffer is full, ends with \0, and doesn't
+        # end with \n\0.
+        strlen = 0
+        while raw_buf[strlen] != '\0':
+            strlen += 1
+        if (strlen == BASE_LINE_SIZE - 1 and
+              raw_buf[BASE_LINE_SIZE - 2] != '\n'):
+            return -1    # overflow!
+        # common case
+        return strlen
+
     def readline(self):
         if self.ll_file:
             raw_buf, gc_buf = rffi.alloc_buffer(BASE_LINE_SIZE)
             try:
-                result = c_fgets(raw_buf, BASE_LINE_SIZE, self.ll_file)
-                if not result:
-                    if c_feof(self.ll_file):   # ok
-                        return ''
-                    errno = c_ferror(self.ll_file)
-                    raise OSError(errno, os.strerror(errno))
+                c = self._readline1(raw_buf)
+                if c >= 0:
+                    return rffi.str_from_buffer(raw_buf, gc_buf,
+                                                BASE_LINE_SIZE, c)
                 #
-                # Assume that fgets() works as documented, and additionally
-                # never writes beyond the final \0, which the CPython
-                # fileobject.c says appears to be the case everywhere.
-                # The only case where the buffer was not big enough is the
-                # case where the buffer is full, ends with \0, and doesn't
-                # end with \n\0.
-                strlen = 0
-                while raw_buf[strlen] != '\0':
-                    strlen += 1
+                # this is the rare case: the line is longer than BASE_LINE_SIZE
+                s = StringBuilder()
+                while True:
+                    s.append_charpsize(raw_buf, BASE_LINE_SIZE - 1)
+                    c = self._readline1(raw_buf)
+                    if c >= 0:
+                        break
                 #
-                if (strlen != BASE_LINE_SIZE - 1 or
-                      raw_buf[BASE_LINE_SIZE - 2] == '\n'):
-                    # common case
-                    return rffi.str_from_buffer(raw_buf, gc_buf,
-                                                BASE_LINE_SIZE, strlen)
-                #
-                XXX
-
+                s.append_charpsize(raw_buf, c)
+                return s.build()
             finally:
                 rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
         raise ValueError("I/O operation on closed file")
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -147,15 +147,30 @@
         j = 0
         expected = []
         with open(fname, 'w') as f:
-            for i in range(50):
-                s = ''.join([chr(32+(k&63)) for k in range(j, j + i*7)])
+            for i in range(150):
+                s = ''.join([chr(32+(k&63)) for k in range(j, j + i)])
+                j += 1
                 print >> f, s
-            f.write('no newline')
         expected = open(fname).readlines()
+        expected += ['', '']
+        assert len(expected) == 152
 
         f = rfile.create_file(fname, 'r')
-        got = []
-        for j in range(53):
-            got.append(f.readline())
+        for j in range(152):
+            got = f.readline()
+            assert got == expected[j]
         f.close()
-        assert got == expected + ['', '']
+
+    def test_readline_without_eol_at_the_end(self):
+        fname = str(self.tmpdir.join('file_readline_without_eol_at_the_end'))
+        for n in [1, 10, 97, 98, 99, 100, 101, 102, 103, 150]:
+            s = ''.join([chr(32+(k&63)) for k in range(n)])
+            with open(fname, 'wb') as f:
+                f.write(s)
+
+            f = rfile.create_file(fname, 'r')
+            got = f.readline()
+            assert got == s
+            got = f.readline()
+            assert got == ''
+            f.close()

From noreply at buildbot.pypy.org  Tue Oct  8 16:56:16 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 16:56:16 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: Implement RFile.readline(), step 1
Message-ID: <20131008145616.3A1711C026D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fileops2
Changeset: r67206:b3affc565846
Date: 2013-10-08 16:43 +0200
http://bitbucket.org/pypy/pypy/changeset/b3affc565846/

Log:	Implement RFile.readline(), step 1

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -43,8 +43,11 @@
 c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], lltype.Signed)
 c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT)
 c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT)
+c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, lltype.Ptr(FILE)],
+                     rffi.CCHARP)
 
 BASE_BUF_SIZE = 4096
+BASE_LINE_SIZE = 1000
 
 def create_file(filename, mode="r", buffering=-1):
     assert buffering == -1
@@ -110,6 +113,7 @@
                 raise OSError(errno, os.strerror(errno))
 
     def read(self, size=-1):
+        # XXX CPython uses a more delicate logic here
         ll_file = self.ll_file
         if not ll_file:
             raise ValueError("I/O operation on closed file")
@@ -190,3 +194,35 @@
     def __del__(self):
         self.close()
 
+    def readline(self):
+        if self.ll_file:
+            raw_buf, gc_buf = rffi.alloc_buffer(BASE_LINE_SIZE)
+            try:
+                result = c_fgets(raw_buf, BASE_LINE_SIZE, self.ll_file)
+                if not result:
+                    if c_feof(self.ll_file):   # ok
+                        return ''
+                    errno = c_ferror(self.ll_file)
+                    raise OSError(errno, os.strerror(errno))
+                #
+                # Assume that fgets() works as documented, and additionally
+                # never writes beyond the final \0, which the CPython
+                # fileobject.c says appears to be the case everywhere.
+                # The only case where the buffer was not big enough is the
+                # case where the buffer is full, ends with \0, and doesn't
+                # end with \n\0.
+                strlen = 0
+                while raw_buf[strlen] != '\0':
+                    strlen += 1
+                #
+                if (strlen != BASE_LINE_SIZE - 1 or
+                      raw_buf[BASE_LINE_SIZE - 2] == '\n'):
+                    # common case
+                    return rffi.str_from_buffer(raw_buf, gc_buf,
+                                                BASE_LINE_SIZE, strlen)
+                #
+                XXX
+
+            finally:
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+        raise ValueError("I/O operation on closed file")
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -135,3 +135,27 @@
 
         f()
         self.interpret(f, [])
+
+
+class TestDirect:
+    def setup_class(cls):
+        cls.tmpdir = udir.join('test_rfile_direct')
+        cls.tmpdir.ensure(dir=True)
+
+    def test_readline(self):
+        fname = str(self.tmpdir.join('file_readline'))
+        j = 0
+        expected = []
+        with open(fname, 'w') as f:
+            for i in range(50):
+                s = ''.join([chr(32+(k&63)) for k in range(j, j + i*7)])
+                print >> f, s
+            f.write('no newline')
+        expected = open(fname).readlines()
+
+        f = rfile.create_file(fname, 'r')
+        got = []
+        for j in range(53):
+            got.append(f.readline())
+        f.close()
+        assert got == expected + ['', '']

From noreply at buildbot.pypy.org  Tue Oct  8 16:57:10 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 16:57:10 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: More tests
Message-ID: <20131008145710.064931C026D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: fileops2
Changeset: r67208:ccfd00cd26be
Date: 2013-10-08 16:56 +0200
http://bitbucket.org/pypy/pypy/changeset/ccfd00cd26be/

Log:	More tests

diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -147,23 +147,24 @@
         j = 0
         expected = []
         with open(fname, 'w') as f:
-            for i in range(150):
+            for i in range(250):
                 s = ''.join([chr(32+(k&63)) for k in range(j, j + i)])
                 j += 1
                 print >> f, s
         expected = open(fname).readlines()
         expected += ['', '']
-        assert len(expected) == 152
+        assert len(expected) == 252
 
         f = rfile.create_file(fname, 'r')
-        for j in range(152):
+        for j in range(252):
             got = f.readline()
             assert got == expected[j]
         f.close()
 
     def test_readline_without_eol_at_the_end(self):
         fname = str(self.tmpdir.join('file_readline_without_eol_at_the_end'))
-        for n in [1, 10, 97, 98, 99, 100, 101, 102, 103, 150]:
+        for n in [1, 10, 97, 98, 99, 100, 101, 102, 103, 150,
+                  196, 197, 198, 199, 200, 201, 202, 203, 204, 250]:
             s = ''.join([chr(32+(k&63)) for k in range(n)])
             with open(fname, 'wb') as f:
                 f.write(s)

From noreply at buildbot.pypy.org  Tue Oct  8 16:58:42 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 16:58:42 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: merge default
Message-ID: <20131008145842.AA3811C026D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67209:ad563885c46f
Date: 2013-10-07 17:33 +0200
http://bitbucket.org/pypy/pypy/changeset/ad563885c46f/

Log:	merge default

diff --git a/rpython/rlib/rpath.py b/rpython/rlib/rpath.py
--- a/rpython/rlib/rpath.py
+++ b/rpython/rlib/rpath.py
@@ -61,3 +61,65 @@
     except os.error:
         return False
     return True
+
+
+import os
+from os.path import isabs, islink, abspath, normpath
+
+def join(a, p):
+    """Join two or more pathname components, inserting '/' as needed.
+    If any component is an absolute path, all previous path components
+    will be discarded.  An empty last part will result in a path that
+    ends with a separator."""
+    path = a
+    for b in p:
+        if b.startswith('/'):
+            path = b
+        elif path == '' or path.endswith('/'):
+            path +=  b
+        else:
+            path += '/' + b
+    return path
+
+def realpath(filename):
+    """Return the canonical path of the specified filename, eliminating any
+symbolic links encountered in the path."""
+    if isabs(filename):
+        bits = ['/'] + filename.split('/')[1:]
+    else:
+        bits = [''] + filename.split('/')
+
+    for i in range(2, len(bits)+1):
+        component = join(bits[0], bits[1:i])
+        # Resolve symbolic links.
+        if islink(component):
+            resolved = _resolve_link(component)
+            if resolved is None:
+                # Infinite loop -- return original component + rest of the path
+                return abspath(join(component, bits[i:]))
+            else:
+                newpath = join(resolved, bits[i:])
+                return realpath(newpath)
+
+    return abspath(filename)
+
+
+def _resolve_link(path):
+    """Internal helper function.  Takes a path and follows symlinks
+    until we either arrive at something that isn't a symlink, or
+    encounter a path we've seen before (meaning that there's a loop).
+    """
+    paths_seen = {}
+    while islink(path):
+        if path in paths_seen:
+            # Already seen this path, so we must have a symlink loop
+            return None
+        paths_seen[path] = None
+        # Resolve where the link points to
+        resolved = os.readlink(path)
+        if not isabs(resolved):
+            dir = dirname(path)
+            path = normpath(join(dir, [resolved]))
+        else:
+            path = normpath(resolved)
+    return path

From noreply at buildbot.pypy.org  Tue Oct  8 16:58:43 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 16:58:43 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: oops
Message-ID: <20131008145843.D476A1C026D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67210:3c2f40e66d67
Date: 2013-10-07 17:51 +0200
http://bitbucket.org/pypy/pypy/changeset/3c2f40e66d67/

Log:	oops

diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -1685,7 +1685,7 @@
         def tmpnam_llimpl():
             return rffi.charp2str(os_tmpnam(lltype.nullptr(rffi.CCHARP.TO)))
 
-        return extdef([], None, llimpl=tmpnam_llimpl,
+        return extdef([], str, llimpl=tmpnam_llimpl,
                       export_name="ll_os.ll_os_tmpnam")
 
 # --------------------------- os.stat & variants ---------------------------

From noreply at buildbot.pypy.org  Tue Oct  8 16:58:45 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 16:58:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: getslice support for bytearrays
Message-ID: <20131008145845.247DA1C026D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67211:99817a6f0494
Date: 2013-10-07 18:03 +0200
http://bitbucket.org/pypy/pypy/changeset/99817a6f0494/

Log:	getslice support for bytearrays

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -10,7 +10,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeFloat, SomeIterator,
     SomePBC, SomeTypedAddressAccess, SomeAddress, SomeType, s_ImpossibleValue,
     s_Bool, s_None, unionof, missing_operation, add_knowntypedata,
-    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString)
+    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper
 from rpython.annotator import builtin
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
@@ -520,6 +520,11 @@
     op_contains.can_only_throw = []
 
 
+class __extend__(SomeByteArray):
+    def getslice(ba, s_start, s_stop):
+        check_negative_slice(s_start, s_stop)
+        return SomeByteArray()
+
 class __extend__(SomeUnicodeString):
     def method_encode(uni, s_enc):
         if not s_enc.is_constant():
diff --git a/rpython/rtyper/lltypesystem/rbytearray.py b/rpython/rtyper/lltypesystem/rbytearray.py
--- a/rpython/rtyper/lltypesystem/rbytearray.py
+++ b/rpython/rtyper/lltypesystem/rbytearray.py
@@ -16,6 +16,9 @@
                                                                   lltype.Char,
                                                                   'bytearray_from_str')
 
+def _empty_bytearray():
+    return empty
+
 BYTEARRAY.become(lltype.GcStruct('rpy_bytearray',
                  ('chars', lltype.Array(lltype.Char)), adtmeths={
     'malloc' : lltype.staticAdtMethod(mallocbytearray),
@@ -23,8 +26,11 @@
     'copy_contents_from_str': lltype.staticAdtMethod(
                                          copy_bytearray_contents_from_str),
     'length': rstr.LLHelpers.ll_length,
+    'empty': lltype.staticAdtMethod(_empty_bytearray),
 }))
 
+empty = lltype.malloc(BYTEARRAY, 0, immortal=True)
+
 class LLHelpers(rstr.LLHelpers):
     @classmethod
     def ll_strsetitem(cls, s, i, item):
diff --git a/rpython/rtyper/test/test_rbytearray.py b/rpython/rtyper/test/test_rbytearray.py
--- a/rpython/rtyper/test/test_rbytearray.py
+++ b/rpython/rtyper/test/test_rbytearray.py
@@ -50,3 +50,10 @@
 
         ll_res = self.interpret(f, [123])
         assert hlstr(ll_res) == "123"
+
+    def test_getslice(self):
+        def f(x):
+            return str(bytearray(str(x))[1:2])
+
+        ll_res = self.interpret(f, [123])
+        assert hlstr(ll_res) == "2"

From noreply at buildbot.pypy.org  Tue Oct  8 16:58:46 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 16:58:46 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: merge default
Message-ID: <20131008145846.529551C026D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67212:122718579c76
Date: 2013-10-07 18:04 +0200
http://bitbucket.org/pypy/pypy/changeset/122718579c76/

Log:	merge default

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -10,7 +10,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeFloat, SomeIterator,
     SomePBC, SomeTypedAddressAccess, SomeAddress, SomeType, s_ImpossibleValue,
     s_Bool, s_None, unionof, missing_operation, add_knowntypedata,
-    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString)
+    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper
 from rpython.annotator import builtin
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
@@ -520,6 +520,11 @@
     op_contains.can_only_throw = []
 
 
+class __extend__(SomeByteArray):
+    def getslice(ba, s_start, s_stop):
+        check_negative_slice(s_start, s_stop)
+        return SomeByteArray()
+
 class __extend__(SomeUnicodeString):
     def method_encode(uni, s_enc):
         if not s_enc.is_constant():
diff --git a/rpython/rtyper/lltypesystem/rbytearray.py b/rpython/rtyper/lltypesystem/rbytearray.py
--- a/rpython/rtyper/lltypesystem/rbytearray.py
+++ b/rpython/rtyper/lltypesystem/rbytearray.py
@@ -16,6 +16,9 @@
                                                                   lltype.Char,
                                                                   'bytearray_from_str')
 
+def _empty_bytearray():
+    return empty
+
 BYTEARRAY.become(lltype.GcStruct('rpy_bytearray',
                  ('chars', lltype.Array(lltype.Char)), adtmeths={
     'malloc' : lltype.staticAdtMethod(mallocbytearray),
@@ -23,8 +26,11 @@
     'copy_contents_from_str': lltype.staticAdtMethod(
                                          copy_bytearray_contents_from_str),
     'length': rstr.LLHelpers.ll_length,
+    'empty': lltype.staticAdtMethod(_empty_bytearray),
 }))
 
+empty = lltype.malloc(BYTEARRAY, 0, immortal=True)
+
 class LLHelpers(rstr.LLHelpers):
     @classmethod
     def ll_strsetitem(cls, s, i, item):
diff --git a/rpython/rtyper/test/test_rbytearray.py b/rpython/rtyper/test/test_rbytearray.py
--- a/rpython/rtyper/test/test_rbytearray.py
+++ b/rpython/rtyper/test/test_rbytearray.py
@@ -50,3 +50,10 @@
 
         ll_res = self.interpret(f, [123])
         assert hlstr(ll_res) == "123"
+
+    def test_getslice(self):
+        def f(x):
+            return str(bytearray(str(x))[1:2])
+
+        ll_res = self.interpret(f, [123])
+        assert hlstr(ll_res) == "2"

From noreply at buildbot.pypy.org  Tue Oct  8 16:58:47 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 16:58:47 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: Move the getting graph to block itself
Message-ID: <20131008145847.924B91C026D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67213:23f91dbd0100
Date: 2013-10-08 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/23f91dbd0100/

Log:	Move the getting graph to block itself

diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -252,6 +252,23 @@
         from rpython.translator.tool.graphpage import try_show
         try_show(self)
 
+    def get_graph(self):
+        import gc
+        pending = [self]   # pending blocks
+        seen = {self: True, None: True}
+        for x in pending:
+            for y in gc.get_referrers(self):
+                if isinstance(y, FunctionGraph):
+                    return y
+                elif isinstance(y, Link):
+                    block = y.prevblock
+                    if block not in seen:
+                        pending.append(block)
+                        seen[block] = True
+                elif isinstance(y, dict):
+                    pending.append(y)   # go back from the dict to the real obj
+        return pending
+
     view = show
 
 
diff --git a/rpython/translator/tool/graphpage.py b/rpython/translator/tool/graphpage.py
--- a/rpython/translator/tool/graphpage.py
+++ b/rpython/translator/tool/graphpage.py
@@ -200,7 +200,7 @@
                 dotgen.emit_edge(nameof(cdef), nameof(prevcdef), color="red")
             prevcdef = cdef
             cdef = cdef.basedef
-        
+
         self.source = dotgen.generate(target=None)
 
     def followlink(self, name):
@@ -224,7 +224,7 @@
         dotgen.emit('mclimit=15.0')
 
         self.do_compute(dotgen, *args, **kwds)
-        
+
         self.source = dotgen.generate(target=None)
 
         # link the function names to the individual flow graphs
@@ -264,7 +264,7 @@
                 data = self.labelof(classdef, classdef.shortname)
                 dotgen.emit_node(nameof(classdef), label=data, shape="box")
                 dotgen.emit_edge(nameof(classdef.basedef), nameof(classdef))
-             
+
     def labelof(self, obj, objname):
         name = objname
         i = 1
@@ -409,22 +409,11 @@
     elif isinstance(obj, Link):
         try_show(obj.prevblock)
     elif isinstance(obj, Block):
-        import gc
-        pending = [obj]   # pending blocks
-        seen = {obj: True, None: True}
-        for x in pending:
-            for y in gc.get_referrers(x):
-                if isinstance(y, FunctionGraph):
-                    y.show()
-                    return
-                elif isinstance(y, Link):
-                    block = y.prevblock
-                    if block not in seen:
-                        pending.append(block)
-                        seen[block] = True
-                elif isinstance(y, dict):
-                    pending.append(y)   # go back from the dict to the real obj
-        graph = IncompleteGraph(pending)
+        graph = obj.get_graph()
+        if isinstance(graph, FunctionGraph):
+            graph.show()
+            return
+        graph = IncompleteGraph(graph)
         SingleGraphPage(graph).display()
     else:
         raise TypeError("try_show(%r object)" % (type(obj).__name__,))
@@ -449,7 +438,7 @@
                         seen[block] = True
         return pending
     else:
-        raise TypeError("try_get_functiongraph(%r object)" % (type(obj).__name__,))    
+        raise TypeError("try_get_functiongraph(%r object)" % (type(obj).__name__,))
 
 class IncompleteGraph:
     name = '(incomplete graph)'

From noreply at buildbot.pypy.org  Tue Oct  8 16:58:48 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 16:58:48 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: oops
Message-ID: <20131008145848.BBEB01C026D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67214:90b0c8837a36
Date: 2013-10-08 15:00 +0200
http://bitbucket.org/pypy/pypy/changeset/90b0c8837a36/

Log:	oops

diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -257,7 +257,7 @@
         pending = [self]   # pending blocks
         seen = {self: True, None: True}
         for x in pending:
-            for y in gc.get_referrers(self):
+            for y in gc.get_referrers(x):
                 if isinstance(y, FunctionGraph):
                     return y
                 elif isinstance(y, Link):

From noreply at buildbot.pypy.org  Tue Oct  8 16:58:49 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 16:58:49 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: merge
Message-ID: <20131008145849.D81F21C026D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67215:70dcd6a26b2c
Date: 2013-10-08 16:56 +0200
http://bitbucket.org/pypy/pypy/changeset/70dcd6a26b2c/

Log:	merge

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -43,8 +43,11 @@
 c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], lltype.Signed)
 c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT)
 c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT)
+c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, lltype.Ptr(FILE)],
+                     rffi.CCHARP)
 
 BASE_BUF_SIZE = 4096
+BASE_LINE_SIZE = 100
 
 def create_file(filename, mode="r", buffering=-1):
     assert buffering == -1
@@ -110,6 +113,7 @@
                 raise OSError(errno, os.strerror(errno))
 
     def read(self, size=-1):
+        # XXX CPython uses a more delicate logic here
         ll_file = self.ll_file
         if not ll_file:
             raise ValueError("I/O operation on closed file")
@@ -190,3 +194,48 @@
     def __del__(self):
         self.close()
 
+    def _readline1(self, raw_buf):
+        result = c_fgets(raw_buf, BASE_LINE_SIZE, self.ll_file)
+        if not result:
+            if c_feof(self.ll_file):   # ok
+                return 0
+            errno = c_ferror(self.ll_file)
+            raise OSError(errno, os.strerror(errno))
+        #
+        # Assume that fgets() works as documented, and additionally
+        # never writes beyond the final \0, which the CPython
+        # fileobject.c says appears to be the case everywhere.
+        # The only case where the buffer was not big enough is the
+        # case where the buffer is full, ends with \0, and doesn't
+        # end with \n\0.
+        strlen = 0
+        while raw_buf[strlen] != '\0':
+            strlen += 1
+        if (strlen == BASE_LINE_SIZE - 1 and
+              raw_buf[BASE_LINE_SIZE - 2] != '\n'):
+            return -1    # overflow!
+        # common case
+        return strlen
+
+    def readline(self):
+        if self.ll_file:
+            raw_buf, gc_buf = rffi.alloc_buffer(BASE_LINE_SIZE)
+            try:
+                c = self._readline1(raw_buf)
+                if c >= 0:
+                    return rffi.str_from_buffer(raw_buf, gc_buf,
+                                                BASE_LINE_SIZE, c)
+                #
+                # this is the rare case: the line is longer than BASE_LINE_SIZE
+                s = StringBuilder()
+                while True:
+                    s.append_charpsize(raw_buf, BASE_LINE_SIZE - 1)
+                    c = self._readline1(raw_buf)
+                    if c >= 0:
+                        break
+                #
+                s.append_charpsize(raw_buf, c)
+                return s.build()
+            finally:
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+        raise ValueError("I/O operation on closed file")
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -135,3 +135,42 @@
 
         f()
         self.interpret(f, [])
+
+
+class TestDirect:
+    def setup_class(cls):
+        cls.tmpdir = udir.join('test_rfile_direct')
+        cls.tmpdir.ensure(dir=True)
+
+    def test_readline(self):
+        fname = str(self.tmpdir.join('file_readline'))
+        j = 0
+        expected = []
+        with open(fname, 'w') as f:
+            for i in range(150):
+                s = ''.join([chr(32+(k&63)) for k in range(j, j + i)])
+                j += 1
+                print >> f, s
+        expected = open(fname).readlines()
+        expected += ['', '']
+        assert len(expected) == 152
+
+        f = rfile.create_file(fname, 'r')
+        for j in range(152):
+            got = f.readline()
+            assert got == expected[j]
+        f.close()
+
+    def test_readline_without_eol_at_the_end(self):
+        fname = str(self.tmpdir.join('file_readline_without_eol_at_the_end'))
+        for n in [1, 10, 97, 98, 99, 100, 101, 102, 103, 150]:
+            s = ''.join([chr(32+(k&63)) for k in range(n)])
+            with open(fname, 'wb') as f:
+                f.write(s)
+
+            f = rfile.create_file(fname, 'r')
+            got = f.readline()
+            assert got == s
+            got = f.readline()
+            assert got == ''
+            f.close()

From noreply at buildbot.pypy.org  Tue Oct  8 16:58:51 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 16:58:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131008145851.0C83C1C026D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67216:9940c4ed7b91
Date: 2013-10-08 16:56 +0200
http://bitbucket.org/pypy/pypy/changeset/9940c4ed7b91/

Log:	merge

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -10,7 +10,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeFloat, SomeIterator,
     SomePBC, SomeTypedAddressAccess, SomeAddress, SomeType, s_ImpossibleValue,
     s_Bool, s_None, unionof, missing_operation, add_knowntypedata,
-    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString)
+    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper
 from rpython.annotator import builtin
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
@@ -520,6 +520,11 @@
     op_contains.can_only_throw = []
 
 
+class __extend__(SomeByteArray):
+    def getslice(ba, s_start, s_stop):
+        check_negative_slice(s_start, s_stop)
+        return SomeByteArray()
+
 class __extend__(SomeUnicodeString):
     def method_encode(uni, s_enc):
         if not s_enc.is_constant():
diff --git a/rpython/rtyper/lltypesystem/rbytearray.py b/rpython/rtyper/lltypesystem/rbytearray.py
--- a/rpython/rtyper/lltypesystem/rbytearray.py
+++ b/rpython/rtyper/lltypesystem/rbytearray.py
@@ -16,6 +16,9 @@
                                                                   lltype.Char,
                                                                   'bytearray_from_str')
 
+def _empty_bytearray():
+    return empty
+
 BYTEARRAY.become(lltype.GcStruct('rpy_bytearray',
                  ('chars', lltype.Array(lltype.Char)), adtmeths={
     'malloc' : lltype.staticAdtMethod(mallocbytearray),
@@ -23,8 +26,11 @@
     'copy_contents_from_str': lltype.staticAdtMethod(
                                          copy_bytearray_contents_from_str),
     'length': rstr.LLHelpers.ll_length,
+    'empty': lltype.staticAdtMethod(_empty_bytearray),
 }))
 
+empty = lltype.malloc(BYTEARRAY, 0, immortal=True)
+
 class LLHelpers(rstr.LLHelpers):
     @classmethod
     def ll_strsetitem(cls, s, i, item):
diff --git a/rpython/rtyper/test/test_rbytearray.py b/rpython/rtyper/test/test_rbytearray.py
--- a/rpython/rtyper/test/test_rbytearray.py
+++ b/rpython/rtyper/test/test_rbytearray.py
@@ -50,3 +50,10 @@
 
         ll_res = self.interpret(f, [123])
         assert hlstr(ll_res) == "123"
+
+    def test_getslice(self):
+        def f(x):
+            return str(bytearray(str(x))[1:2])
+
+        ll_res = self.interpret(f, [123])
+        assert hlstr(ll_res) == "2"

From noreply at buildbot.pypy.org  Tue Oct  8 16:58:52 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 16:58:52 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: merge
Message-ID: <20131008145852.1C29A1C026D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67217:135dd61a66ae
Date: 2013-10-08 16:57 +0200
http://bitbucket.org/pypy/pypy/changeset/135dd61a66ae/

Log:	merge

diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -147,23 +147,24 @@
         j = 0
         expected = []
         with open(fname, 'w') as f:
-            for i in range(150):
+            for i in range(250):
                 s = ''.join([chr(32+(k&63)) for k in range(j, j + i)])
                 j += 1
                 print >> f, s
         expected = open(fname).readlines()
         expected += ['', '']
-        assert len(expected) == 152
+        assert len(expected) == 252
 
         f = rfile.create_file(fname, 'r')
-        for j in range(152):
+        for j in range(252):
             got = f.readline()
             assert got == expected[j]
         f.close()
 
     def test_readline_without_eol_at_the_end(self):
         fname = str(self.tmpdir.join('file_readline_without_eol_at_the_end'))
-        for n in [1, 10, 97, 98, 99, 100, 101, 102, 103, 150]:
+        for n in [1, 10, 97, 98, 99, 100, 101, 102, 103, 150,
+                  196, 197, 198, 199, 200, 201, 202, 203, 204, 250]:
             s = ''.join([chr(32+(k&63)) for k in range(n)])
             with open(fname, 'wb') as f:
                 f.write(s)

From noreply at buildbot.pypy.org  Tue Oct  8 17:07:14 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue,  8 Oct 2013 17:07:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add issubdtype
Message-ID: <20131008150714.2A88B1C026D@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r67218:8f6cfebce395
Date: 2013-10-08 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/8f6cfebce395/

Log:	Add issubdtype

diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -1,1 +1,75 @@
 from _numpypy.numerictypes import *
+import numpypy
+
+def issubclass_(arg1, arg2):
+    """
+    Determine if a class is a subclass of a second class.
+
+    `issubclass_` is equivalent to the Python built-in ``issubclass``,
+    except that it returns False instead of raising a TypeError is one
+    of the arguments is not a class.
+
+    Parameters
+    ----------
+    arg1 : class
+        Input class. True is returned if `arg1` is a subclass of `arg2`.
+    arg2 : class or tuple of classes.
+        Input class. If a tuple of classes, True is returned if `arg1` is a
+        subclass of any of the tuple elements.
+
+    Returns
+    -------
+    out : bool
+        Whether `arg1` is a subclass of `arg2` or not.
+
+    See Also
+    --------
+    issubsctype, issubdtype, issctype
+
+    Examples
+    --------
+    >>> np.issubclass_(np.int32, np.int)
+    True
+    >>> np.issubclass_(np.int32, np.float)
+    False
+
+    """
+    try:
+        return issubclass(arg1, arg2)
+    except TypeError:
+        return False
+
+def issubdtype(arg1, arg2):
+    """
+    Returns True if first argument is a typecode lower/equal in type hierarchy.
+
+    Parameters
+    ----------
+    arg1, arg2 : dtype_like
+        dtype or string representing a typecode.
+
+    Returns
+    -------
+    out : bool
+
+    See Also
+    --------
+    issubsctype, issubclass_
+    numpy.core.numerictypes : Overview of numpy type hierarchy.
+
+    Examples
+    --------
+    >>> np.issubdtype('S1', str)
+    True
+    >>> np.issubdtype(np.float64, np.float32)
+    False
+
+    """
+    if issubclass_(arg2, generic):
+        return issubclass(numpypy.dtype(arg1).type, arg2)
+    mro = numpypy.dtype(arg2).type.mro()
+    if len(mro) > 1:
+        val = mro[1]
+    else:
+        val = mro[0]
+    return issubclass(numpypy.dtype(arg1).type, val)

From noreply at buildbot.pypy.org  Tue Oct  8 17:14:24 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 17:14:24 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: some more signatures
Message-ID: <20131008151424.6FCC41C0161@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67219:4719ccf619b8
Date: 2013-10-08 17:11 +0200
http://bitbucket.org/pypy/pypy/changeset/4719ccf619b8/

Log:	some more signatures

diff --git a/rpython/rlib/types.py b/rpython/rlib/types.py
--- a/rpython/rlib/types.py
+++ b/rpython/rlib/types.py
@@ -39,8 +39,12 @@
     return model.SomeUnicodeString(no_nul=True)
 
 
-def str():
-    return model.SomeString()
+def str(can_be_None=False):
+    return model.SomeString(can_be_None=can_be_None)
+
+
+def bytearray():
+    return model.SomeByteArray()
 
 
 def str0():

From noreply at buildbot.pypy.org  Tue Oct  8 17:14:25 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 17:14:25 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: revert accidental change
Message-ID: <20131008151425.8E1821C0161@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67220:097e2a00a246
Date: 2013-10-08 17:11 +0200
http://bitbucket.org/pypy/pypy/changeset/097e2a00a246/

Log:	revert accidental change

diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -11,10 +11,10 @@
 
 import py
 # clean up early rpython/_cache
-#try:
-#    py.path.local(cache_dir).remove()
-#except Exception:
-#    pass
+try:
+    py.path.local(cache_dir).remove()
+except Exception:
+    pass
 
 from rpython.config.config import (to_optparse, OptionDescription, BoolOption,
     ArbitraryOption, StrOption, IntOption, Config, ChoiceOption, OptHelpFormatter)

From noreply at buildbot.pypy.org  Tue Oct  8 17:14:26 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 17:14:26 +0200 (CEST)
Subject: [pypy-commit] pypy fileops2: close merged branch
Message-ID: <20131008151426.9E1201C0161@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: fileops2
Changeset: r67221:c985e1edb7da
Date: 2013-10-08 17:12 +0200
http://bitbucket.org/pypy/pypy/changeset/c985e1edb7da/

Log:	close merged branch


From noreply at buildbot.pypy.org  Tue Oct  8 17:14:27 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 17:14:27 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal, arigo) merge fileops2,
 which brings some sanity (and more featureS) to file implementation
 in RPython
Message-ID: <20131008151427.C9A281C0161@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67222:8fd575d74e94
Date: 2013-10-08 17:12 +0200
http://bitbucket.org/pypy/pypy/changeset/8fd575d74e94/

Log:	(fijal, arigo) merge fileops2, which brings some sanity (and more
	featureS) to file implementation in RPython

diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -252,6 +252,23 @@
         from rpython.translator.tool.graphpage import try_show
         try_show(self)
 
+    def get_graph(self):
+        import gc
+        pending = [self]   # pending blocks
+        seen = {self: True, None: True}
+        for x in pending:
+            for y in gc.get_referrers(x):
+                if isinstance(y, FunctionGraph):
+                    return y
+                elif isinstance(y, Link):
+                    block = y.prevblock
+                    if block not in seen:
+                        pending.append(block)
+                        seen[block] = True
+                elif isinstance(y, dict):
+                    pending.append(y)   # go back from the dict to the real obj
+        return pending
+
     view = show
 
 
diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py
--- a/rpython/flowspace/specialcase.py
+++ b/rpython/flowspace/specialcase.py
@@ -1,3 +1,4 @@
+import os
 from rpython.flowspace.model import Constant, const
 
 SPECIAL_CASES = {}
@@ -37,6 +38,18 @@
     return space.frame.do_operation('simple_call', const(isinstance),
             w_instance, w_type)
 
+ at register_flow_sc(open)
+def sc_open(space, *args_w):
+    from rpython.rlib.rfile import create_file
+
+    return space.frame.do_operation("simple_call", const(create_file), *args_w)
+
+ at register_flow_sc(os.tmpfile)
+def sc_os_tmpfile(space):
+    from rpython.rlib.rfile import create_temp_rfile
+
+    return space.frame.do_operation("simple_call", const(create_temp_rfile))
+
 # _________________________________________________________________________
 # a simplified version of the basic printing routines, for RPython programs
 class StdOutBuffer:
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -1,55 +1,241 @@
 
-""" This file makes open() and friends RPython
+""" This file makes open() and friends RPython. Note that RFile should not
+be used directly and instead it's magically appearing each time you call
+python builtin open()
 """
 
 import os
-from rpython.annotator.model import SomeObject, SomeString, SomeInteger
-from rpython.rtyper.extregistry import ExtRegistryEntry
-from rpython.rtyper.extfunc import register_external
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.tool import rffi_platform as platform
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib import rposix
+from rpython.rlib.rstring import StringBuilder
 
-class SomeFile(SomeObject):
-    def method_write(self, s_arg):
-        assert isinstance(s_arg, SomeString)
+eci = ExternalCompilationInfo(includes=['stdio.h', 'unistd.h', 'sys/types.h'])
 
-    def method_read(self, s_arg=None):
-        if s_arg is not None:
-            assert isinstance(s_arg, SomeInteger)
-        return SomeString(can_be_None=False)
+def llexternal(*args):
+    return rffi.llexternal(*args, compilation_info=eci)
 
-    def method_close(self):
-        pass
+FILE = lltype.Struct('FILE') # opaque type maybe
 
-    def method_seek(self, s_arg, s_whence=None):
-        assert isinstance(s_arg, SomeInteger)
-        if s_whence is not None:
-            assert isinstance(s_whence, SomeInteger)
+class CConfig(object):
+    _compilation_info_ = eci
 
-    def rtyper_makekey(self):
-        return self.__class__,
+    off_t = platform.SimpleType('off_t')
 
-    def rtyper_makerepr(self, rtyper):
-        from rpython.rtyper.lltypesystem.rfile import FileRepr
+CC = platform.configure(CConfig)
+OFF_T = CC['off_t']
 
-        return FileRepr(rtyper)
+c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
+c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
+c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
+                                     lltype.Ptr(FILE)], rffi.SIZE_T)
+c_read = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
+                                   lltype.Ptr(FILE)], rffi.SIZE_T)
+c_feof = llexternal('feof', [lltype.Ptr(FILE)], rffi.INT)
+c_ferror = llexternal('ferror', [lltype.Ptr(FILE)], rffi.INT)
+c_clearerror = llexternal('clearerr', [lltype.Ptr(FILE)], lltype.Void)
+c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT],
+                          rffi.INT)
+c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
+c_fileno = llexternal('fileno', [lltype.Ptr(FILE)], rffi.INT)
+c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], lltype.Signed)
+c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT)
+c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT)
+c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, lltype.Ptr(FILE)],
+                     rffi.CCHARP)
 
-class FileEntry(ExtRegistryEntry):
-    _about_ = open
+BASE_BUF_SIZE = 4096
+BASE_LINE_SIZE = 100
 
-    def compute_result_annotation(self, s_name, s_mode=None):
-        assert isinstance(s_name, SomeString)
-        if s_mode is not None:
-            assert isinstance(s_mode, SomeString)
-        return SomeFile()
+def create_file(filename, mode="r", buffering=-1):
+    assert buffering == -1
+    assert filename is not None
+    assert mode is not None
+    ll_name = rffi.str2charp(filename)
+    try:
+        ll_mode = rffi.str2charp(mode)
+        try:
+            ll_f = c_open(ll_name, ll_mode)
+            if not ll_f:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+        finally:
+            lltype.free(ll_mode, flavor='raw')
+    finally:
+        lltype.free(ll_name, flavor='raw')
+    return RFile(ll_f)
 
-    def specialize_call(self, hop):
-        return hop.r_result.rtype_constructor(hop)
+def create_temp_rfile():
+    res = c_tmpfile()
+    if not res:
+        errno = rposix.get_errno()
+        raise OSError(errno, os.strerror(errno))
+    return RFile(res)
 
-class OSTempfileEntry(ExtRegistryEntry):
-    _about_ = os.tmpfile
+class RFile(object):
+    def __init__(self, ll_file):
+        self.ll_file = ll_file
 
-    def compute_result_annotation(self):
-        return SomeFile()
+    def write(self, value):
+        assert value is not None
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        assert value is not None
+        ll_value = rffi.get_nonmovingbuffer(value)
+        try:
+            # note that since we got a nonmoving buffer, it is either raw
+            # or already cannot move, so the arithmetics below are fine
+            total_bytes = 0
+            ll_current = ll_value
+            while total_bytes < len(value):
+                bytes = c_write(ll_current, 1, len(value) - r_uint(total_bytes),
+                                ll_file)
+                if bytes == 0:
+                    errno = rposix.get_errno()
+                    raise OSError(errno, os.strerror(errno))
+                total_bytes += bytes
+                ll_current = rffi.cast(rffi.CCHARP,
+                                       rffi.cast(lltype.Unsigned, ll_value) +
+                                       total_bytes)
+        finally:
+            rffi.free_nonmovingbuffer(value, ll_value)
 
-    def specialize_call(self, hop):
-        return hop.r_result.rtype_tempfile(hop)
+    def close(self):
+        if self.ll_file:
+            # double close is allowed
+            res = c_close(self.ll_file)
+            self.ll_file = lltype.nullptr(FILE)
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
 
+    def read(self, size=-1):
+        # XXX CPython uses a more delicate logic here
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        if size < 0:
+            # read the entire contents
+            buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
+            try:
+                s = StringBuilder()
+                while True:
+                    returned_size = c_read(buf, 1, BASE_BUF_SIZE, ll_file)
+                    if returned_size == 0:
+                        if c_feof(ll_file):
+                            # ok, finished
+                            return s.build()
+                        errno = c_ferror(ll_file)
+                        c_clearerror(ll_file)
+                        raise OSError(errno, os.strerror(errno))
+                    s.append_charpsize(buf, returned_size)
+            finally:
+                lltype.free(buf, flavor='raw')
+        else:
+            raw_buf, gc_buf = rffi.alloc_buffer(size)
+            try:
+                returned_size = c_read(raw_buf, 1, size, ll_file)
+                if returned_size == 0:
+                    if not c_feof(ll_file):
+                        errno = c_ferror(ll_file)
+                        raise OSError(errno, os.strerror(errno))
+                s = rffi.str_from_buffer(raw_buf, gc_buf, size,
+                                         rffi.cast(lltype.Signed, returned_size))
+            finally:
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+            return s
+
+    def seek(self, pos, whence=0):
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        res = c_fseek(ll_file, pos, whence)
+        if res == -1:
+            errno = rposix.get_errno()
+            raise OSError(errno, os.strerror(errno))
+
+    def fileno(self):
+        if self.ll_file:
+            return intmask(c_fileno(self.ll_file))
+        raise ValueError("I/O operation on closed file")
+
+    def tell(self):
+        if self.ll_file:
+            res = intmask(c_ftell(self.ll_file))
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return res
+        raise ValueError("I/O operation on closed file")
+
+    def flush(self):
+        if self.ll_file:
+            res = c_fflush(self.ll_file)
+            if res != 0:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return
+        raise ValueError("I/O operation on closed file")
+
+    def truncate(self, arg=-1):
+        if self.ll_file:
+            if arg == -1:
+                arg = self.tell()
+            res = c_ftruncate(self.fileno(), arg)
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return
+        raise ValueError("I/O operation on closed file")
+
+    def __del__(self):
+        self.close()
+
+    def _readline1(self, raw_buf):
+        result = c_fgets(raw_buf, BASE_LINE_SIZE, self.ll_file)
+        if not result:
+            if c_feof(self.ll_file):   # ok
+                return 0
+            errno = c_ferror(self.ll_file)
+            raise OSError(errno, os.strerror(errno))
+        #
+        # Assume that fgets() works as documented, and additionally
+        # never writes beyond the final \0, which the CPython
+        # fileobject.c says appears to be the case everywhere.
+        # The only case where the buffer was not big enough is the
+        # case where the buffer is full, ends with \0, and doesn't
+        # end with \n\0.
+        strlen = 0
+        while raw_buf[strlen] != '\0':
+            strlen += 1
+        if (strlen == BASE_LINE_SIZE - 1 and
+              raw_buf[BASE_LINE_SIZE - 2] != '\n'):
+            return -1    # overflow!
+        # common case
+        return strlen
+
+    def readline(self):
+        if self.ll_file:
+            raw_buf, gc_buf = rffi.alloc_buffer(BASE_LINE_SIZE)
+            try:
+                c = self._readline1(raw_buf)
+                if c >= 0:
+                    return rffi.str_from_buffer(raw_buf, gc_buf,
+                                                BASE_LINE_SIZE, c)
+                #
+                # this is the rare case: the line is longer than BASE_LINE_SIZE
+                s = StringBuilder()
+                while True:
+                    s.append_charpsize(raw_buf, BASE_LINE_SIZE - 1)
+                    c = self._readline1(raw_buf)
+                    if c >= 0:
+                        break
+                #
+                s.append_charpsize(raw_buf, c)
+                return s.build()
+            finally:
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+        raise ValueError("I/O operation on closed file")
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -78,3 +78,100 @@
 
         f()
         self.interpret(f, [])
+
+    def test_fileno(self):
+        fname = str(self.tmpdir.join('file_5'))
+
+        def f():
+            f = open(fname, "w")
+            try:
+                return f.fileno()
+            finally:
+                f.close()
+
+        res = self.interpret(f, [])
+        assert res > 2
+
+    def test_tell(self):
+        fname = str(self.tmpdir.join('file_tell'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            try:
+                return f.tell()
+            finally:
+                f.close()
+
+        res = self.interpret(f, [])
+        assert res == 3
+
+    def test_flush(self):
+        fname = str(self.tmpdir.join('file_flush'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            f.flush()
+            f2 = open(fname)
+            assert f2.read() == "xyz"
+            f2.close()
+            f.close()
+
+        self.interpret(f, [])
+
+    def test_truncate(self):
+        fname = str(self.tmpdir.join('file_trunc'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            f.seek(0)
+            f.truncate(2)
+            f.close()
+            f2 = open(fname)
+            assert f2.read() == "xy"
+            f2.close()
+
+        f()
+        self.interpret(f, [])
+
+
+class TestDirect:
+    def setup_class(cls):
+        cls.tmpdir = udir.join('test_rfile_direct')
+        cls.tmpdir.ensure(dir=True)
+
+    def test_readline(self):
+        fname = str(self.tmpdir.join('file_readline'))
+        j = 0
+        expected = []
+        with open(fname, 'w') as f:
+            for i in range(250):
+                s = ''.join([chr(32+(k&63)) for k in range(j, j + i)])
+                j += 1
+                print >> f, s
+        expected = open(fname).readlines()
+        expected += ['', '']
+        assert len(expected) == 252
+
+        f = rfile.create_file(fname, 'r')
+        for j in range(252):
+            got = f.readline()
+            assert got == expected[j]
+        f.close()
+
+    def test_readline_without_eol_at_the_end(self):
+        fname = str(self.tmpdir.join('file_readline_without_eol_at_the_end'))
+        for n in [1, 10, 97, 98, 99, 100, 101, 102, 103, 150,
+                  196, 197, 198, 199, 200, 201, 202, 203, 204, 250]:
+            s = ''.join([chr(32+(k&63)) for k in range(n)])
+            with open(fname, 'wb') as f:
+                f.write(s)
+
+            f = rfile.create_file(fname, 'r')
+            got = f.readline()
+            assert got == s
+            got = f.readline()
+            assert got == ''
+            f.close()
diff --git a/rpython/rlib/types.py b/rpython/rlib/types.py
--- a/rpython/rlib/types.py
+++ b/rpython/rlib/types.py
@@ -39,8 +39,12 @@
     return model.SomeUnicodeString(no_nul=True)
 
 
-def str():
-    return model.SomeString()
+def str(can_be_None=False):
+    return model.SomeString(can_be_None=can_be_None)
+
+
+def bytearray():
+    return model.SomeByteArray()
 
 
 def str0():
diff --git a/rpython/rtyper/lltypesystem/rfile.py b/rpython/rtyper/lltypesystem/rfile.py
deleted file mode 100644
--- a/rpython/rtyper/lltypesystem/rfile.py
+++ /dev/null
@@ -1,195 +0,0 @@
-
-import os
-from rpython.rlib import rposix
-from rpython.rlib.rarithmetic import r_uint
-from rpython.annotator import model as annmodel
-from rpython.rtyper.rtyper import Repr
-from rpython.rlib.rstring import StringBuilder
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
-from rpython.rtyper.lltypesystem.rstr import string_repr, STR
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.annlowlevel import hlstr
-from rpython.rtyper.lltypesystem.lloperation import llop
-
-FILE = lltype.Struct('FILE') # opaque type maybe
-FILE_WRAPPER = lltype.GcStruct("FileWrapper", ('file', lltype.Ptr(FILE)))
-
-eci = ExternalCompilationInfo(includes=['stdio.h'])
-
-def llexternal(*args):
-    return rffi.llexternal(*args, compilation_info=eci)
-
-c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
-c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
-c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
-                                     lltype.Ptr(FILE)], rffi.SIZE_T)
-c_read = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
-                                   lltype.Ptr(FILE)], rffi.SIZE_T)
-c_feof = llexternal('feof', [lltype.Ptr(FILE)], rffi.INT)
-c_ferror = llexternal('ferror', [lltype.Ptr(FILE)], rffi.INT)
-c_clearerror = llexternal('clearerr', [lltype.Ptr(FILE)], lltype.Void)
-c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT],
-                          rffi.INT)
-c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
-
-def ll_open(name, mode):
-    file_wrapper = lltype.malloc(FILE_WRAPPER)
-    ll_name = rffi.str2charp(name)
-    ll_mode = rffi.str2charp(mode)
-    try:
-        ll_f = c_open(ll_name, ll_mode)
-        if not ll_f:
-            errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
-        file_wrapper.file = ll_f
-    finally:
-        lltype.free(ll_name, flavor='raw')
-        lltype.free(ll_mode, flavor='raw')
-    return file_wrapper
-
-def ll_tmpfile():
-    file_wrapper = lltype.malloc(FILE_WRAPPER)
-    res = c_tmpfile()
-    if not res:
-        errno = rposix.get_errno()
-        raise OSError(errno, os.strerror(errno))
-    file_wrapper.file = res
-    return file_wrapper
-
-def ll_write(file_wrapper, value):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    value = hlstr(value)
-    assert value is not None
-    ll_value = rffi.get_nonmovingbuffer(value)
-    try:
-        # note that since we got a nonmoving buffer, it is either raw
-        # or already cannot move, so the arithmetics below are fine
-        total_bytes = 0
-        ll_current = ll_value
-        while total_bytes < len(value):
-            bytes = c_write(ll_current, 1, len(value) - r_uint(total_bytes),
-                            ll_file)
-            if bytes == 0:
-                errno = rposix.get_errno()
-                raise OSError(errno, os.strerror(errno))
-            total_bytes += bytes
-            ll_current = rffi.cast(rffi.CCHARP,
-                                   rffi.cast(lltype.Unsigned, ll_value) +
-                                   total_bytes)
-    finally:
-        rffi.free_nonmovingbuffer(value, ll_value)
-
-BASE_BUF_SIZE = 4096
-
-def ll_read(file_wrapper, size):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    if size < 0:
-        # read the entire contents
-        buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
-        try:
-            s = StringBuilder()
-            while True:
-                returned_size = c_read(buf, 1, BASE_BUF_SIZE, ll_file)
-                if returned_size == 0:
-                    if c_feof(ll_file):
-                        # ok, finished
-                        return s.build()
-                    errno = c_ferror(ll_file)
-                    c_clearerror(ll_file)
-                    raise OSError(errno, os.strerror(errno))
-                s.append_charpsize(buf, returned_size)
-        finally:
-            lltype.free(buf, flavor='raw')
-    else:
-        raw_buf, gc_buf = rffi.alloc_buffer(size)
-        try:
-            returned_size = c_read(raw_buf, 1, size, ll_file)
-            if returned_size == 0:
-                if not c_feof(ll_file):
-                    errno = c_ferror(ll_file)
-                    raise OSError(errno, os.strerror(errno))
-            s = rffi.str_from_buffer(raw_buf, gc_buf, size,
-                                     rffi.cast(lltype.Signed, returned_size))
-        finally:
-            rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
-        return s
-def ll_seek(file_wrapper, pos, whence):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    res = c_fseek(ll_file, pos, whence)
-    if res == -1:
-        errno = rposix.get_errno()
-        raise OSError(errno, os.strerror(errno))
-
-def ll_close(file_wrapper):
-    if file_wrapper.file:
-        # double close is allowed
-        res = c_close(file_wrapper.file)
-        file_wrapper.file = lltype.nullptr(FILE)
-        if res == -1:
-            errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
-
-class FileRepr(Repr):
-    lowleveltype = lltype.Ptr(FILE_WRAPPER)
-
-    def __init__(self, typer):
-        Repr.__init__(self)
-
-    def rtype_constructor(self, hop):
-        repr = hop.rtyper.getrepr(annmodel.SomeString())
-        arg_0 = hop.inputarg(repr, 0)
-        if len(hop.args_v) == 1:
-            arg_1 = hop.inputconst(string_repr, "r")
-        else:
-            arg_1 = hop.inputarg(repr, 1)
-        hop.exception_is_here()
-        open = hop.rtyper.getannmixlevel().delayedfunction(
-            ll_open, [annmodel.SomeString()] * 2,
-            annmodel.SomePtr(self.lowleveltype))
-        v_open = hop.inputconst(lltype.typeOf(open), open)
-        return hop.genop('direct_call', [v_open, arg_0, arg_1],
-                         resulttype=self)
-
-    def rtype_tempfile(self, hop):
-        tmpfile = hop.rtyper.getannmixlevel().delayedfunction(
-            ll_tmpfile, [], annmodel.SomePtr(self.lowleveltype))
-        v_tmpfile = hop.inputconst(lltype.typeOf(tmpfile), tmpfile)
-        hop.exception_is_here()
-        return hop.genop('direct_call', [v_tmpfile], resulttype=self)
-
-
-    def rtype_method_write(self, hop):
-        args_v = hop.inputargs(self, string_repr)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_write, *args_v)
-
-    def rtype_method_close(self, hop):
-        r_self = hop.inputarg(self, 0)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_close, r_self)
-
-    def rtype_method_read(self, hop):
-        r_self = hop.inputarg(self, 0)
-        if len(hop.args_v) != 2:
-            arg_1 = hop.inputconst(lltype.Signed, -1)
-        else:
-            arg_1 = hop.inputarg(lltype.Signed, 1)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_read, r_self, arg_1)
-
-    def rtype_method_seek(self, hop):
-        r_self = hop.inputarg(self, 0)
-        arg_1 = hop.inputarg(lltype.Signed, 1)
-        if len(hop.args_v) != 3:
-            arg_2 = hop.inputconst(lltype.Signed, os.SEEK_SET)
-        else:
-            arg_2 = hop.inputarg(lltype.Signed, 2)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_seek, r_self, arg_1, arg_2)
-
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -1685,7 +1685,7 @@
         def tmpnam_llimpl():
             return rffi.charp2str(os_tmpnam(lltype.nullptr(rffi.CCHARP.TO)))
 
-        return extdef([], None, llimpl=tmpnam_llimpl,
+        return extdef([], str, llimpl=tmpnam_llimpl,
                       export_name="ll_os.ll_os_tmpnam")
 
 # --------------------------- os.stat & variants ---------------------------
diff --git a/rpython/translator/tool/graphpage.py b/rpython/translator/tool/graphpage.py
--- a/rpython/translator/tool/graphpage.py
+++ b/rpython/translator/tool/graphpage.py
@@ -200,7 +200,7 @@
                 dotgen.emit_edge(nameof(cdef), nameof(prevcdef), color="red")
             prevcdef = cdef
             cdef = cdef.basedef
-        
+
         self.source = dotgen.generate(target=None)
 
     def followlink(self, name):
@@ -224,7 +224,7 @@
         dotgen.emit('mclimit=15.0')
 
         self.do_compute(dotgen, *args, **kwds)
-        
+
         self.source = dotgen.generate(target=None)
 
         # link the function names to the individual flow graphs
@@ -264,7 +264,7 @@
                 data = self.labelof(classdef, classdef.shortname)
                 dotgen.emit_node(nameof(classdef), label=data, shape="box")
                 dotgen.emit_edge(nameof(classdef.basedef), nameof(classdef))
-             
+
     def labelof(self, obj, objname):
         name = objname
         i = 1
@@ -409,22 +409,11 @@
     elif isinstance(obj, Link):
         try_show(obj.prevblock)
     elif isinstance(obj, Block):
-        import gc
-        pending = [obj]   # pending blocks
-        seen = {obj: True, None: True}
-        for x in pending:
-            for y in gc.get_referrers(x):
-                if isinstance(y, FunctionGraph):
-                    y.show()
-                    return
-                elif isinstance(y, Link):
-                    block = y.prevblock
-                    if block not in seen:
-                        pending.append(block)
-                        seen[block] = True
-                elif isinstance(y, dict):
-                    pending.append(y)   # go back from the dict to the real obj
-        graph = IncompleteGraph(pending)
+        graph = obj.get_graph()
+        if isinstance(graph, FunctionGraph):
+            graph.show()
+            return
+        graph = IncompleteGraph(graph)
         SingleGraphPage(graph).display()
     else:
         raise TypeError("try_show(%r object)" % (type(obj).__name__,))
@@ -449,7 +438,7 @@
                         seen[block] = True
         return pending
     else:
-        raise TypeError("try_get_functiongraph(%r object)" % (type(obj).__name__,))    
+        raise TypeError("try_get_functiongraph(%r object)" % (type(obj).__name__,))
 
 class IncompleteGraph:
     name = '(incomplete graph)'

From noreply at buildbot.pypy.org  Tue Oct  8 17:14:28 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  8 Oct 2013 17:14:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131008151428.ED0361C0161@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67223:6441df636735
Date: 2013-10-08 17:13 +0200
http://bitbucket.org/pypy/pypy/changeset/6441df636735/

Log:	merge

diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -1,1 +1,75 @@
 from _numpypy.numerictypes import *
+import numpypy
+
+def issubclass_(arg1, arg2):
+    """
+    Determine if a class is a subclass of a second class.
+
+    `issubclass_` is equivalent to the Python built-in ``issubclass``,
+    except that it returns False instead of raising a TypeError is one
+    of the arguments is not a class.
+
+    Parameters
+    ----------
+    arg1 : class
+        Input class. True is returned if `arg1` is a subclass of `arg2`.
+    arg2 : class or tuple of classes.
+        Input class. If a tuple of classes, True is returned if `arg1` is a
+        subclass of any of the tuple elements.
+
+    Returns
+    -------
+    out : bool
+        Whether `arg1` is a subclass of `arg2` or not.
+
+    See Also
+    --------
+    issubsctype, issubdtype, issctype
+
+    Examples
+    --------
+    >>> np.issubclass_(np.int32, np.int)
+    True
+    >>> np.issubclass_(np.int32, np.float)
+    False
+
+    """
+    try:
+        return issubclass(arg1, arg2)
+    except TypeError:
+        return False
+
+def issubdtype(arg1, arg2):
+    """
+    Returns True if first argument is a typecode lower/equal in type hierarchy.
+
+    Parameters
+    ----------
+    arg1, arg2 : dtype_like
+        dtype or string representing a typecode.
+
+    Returns
+    -------
+    out : bool
+
+    See Also
+    --------
+    issubsctype, issubclass_
+    numpy.core.numerictypes : Overview of numpy type hierarchy.
+
+    Examples
+    --------
+    >>> np.issubdtype('S1', str)
+    True
+    >>> np.issubdtype(np.float64, np.float32)
+    False
+
+    """
+    if issubclass_(arg2, generic):
+        return issubclass(numpypy.dtype(arg1).type, arg2)
+    mro = numpypy.dtype(arg2).type.mro()
+    if len(mro) > 1:
+        val = mro[1]
+    else:
+        val = mro[0]
+    return issubclass(numpypy.dtype(arg1).type, val)

From noreply at buildbot.pypy.org  Tue Oct  8 18:00:13 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  8 Oct 2013 18:00:13 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Fix: see comments in
	collect_and_reserve()
Message-ID: <20131008160013.8F16F1C026D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67224:8ad8d5aa4c4e
Date: 2013-10-08 17:59 +0200
http://bitbucket.org/pypy/pypy/changeset/8ad8d5aa4c4e/

Log:	Fix: see comments in collect_and_reserve()

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -704,21 +704,27 @@
             self.move_nursery_top(totalsize)
             return prev_result
         self.minor_collection()
-        self.major_collection_step()
         #
-
-        #
-        # The nursery might not be empty now, because of
-        # execute_finalizers().  If it is almost full again,
-        # we need to fix it with another call to minor_collection().
-        if self.nursery_free + totalsize > self.nursery_top:
+        # If the gc_state is not STATE_SCANNING, we're in the middle of
+        # an incremental major collection.  In this case, always progress
+        # one step.  If the gc_state is STATE_SCANNING, wait until there
+        # is too much garbage before starting the next major collection.
+        if (self.gc_state != STATE_SCANNING or
+                    self.get_total_memory_used() >
+                    self.next_major_collection_threshold):
+            self.major_collection_step()
             #
-            if self.nursery_free + totalsize > self.nursery_real_top:
-                self.minor_collection()
-                # then the nursery is empty
-            else:
-                # we just need to clean up a bit more of the nursery
-                self.move_nursery_top(totalsize)
+            # The nursery might not be empty now, because of
+            # execute_finalizers().  If it is almost full again,
+            # we need to fix it with another call to minor_collection().
+            if self.nursery_free + totalsize > self.nursery_top:
+                #
+                if self.nursery_free + totalsize > self.nursery_real_top:
+                    self.minor_collection()
+                    # then the nursery is empty
+                else:
+                    # we just need to clean up a bit more of the nursery
+                    self.move_nursery_top(totalsize)
         #
         result = self.nursery_free
         self.nursery_free = result + totalsize

From noreply at buildbot.pypy.org  Tue Oct  8 18:27:28 2013
From: noreply at buildbot.pypy.org (stefanor)
Date: Tue,  8 Oct 2013 18:27:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: There is no magic __ARM_ARCH__ integer.
 Just check for ARMv4
Message-ID: <20131008162728.E30DF1C1502@cobra.cs.uni-duesseldorf.de>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r67225:c4cd46b31996
Date: 2013-10-08 18:26 +0200
http://bitbucket.org/pypy/pypy/changeset/c4cd46b31996/

Log:	There is no magic __ARM_ARCH__ integer. Just check for ARMv4

diff --git a/rpython/translator/c/src/stacklet/switch_arm_gcc.h b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
--- a/rpython/translator/c/src/stacklet/switch_arm_gcc.h
+++ b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
@@ -1,9 +1,8 @@
-#if __ARM_ARCH__ >= 5
-# define call_reg(x) "blx " #x "\n"
-#elif defined (__ARM_ARCH_4T__)
+#if defined(__ARM_ARCH_4__) || defined (__ARM_ARCH_4T__)
 # define call_reg(x) "mov lr, pc ; bx " #x "\n"
 #else
-# define call_reg(x) "mov lr, pc ; mov pc, " #x "\n"
+/* ARM >= 5 */
+# define call_reg(x) "blx " #x "\n"
 #endif
 
 static void __attribute__((optimize("O3"))) *slp_switch(void *(*save_state)(void*, void*),

From noreply at buildbot.pypy.org  Tue Oct  8 23:33:45 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue,  8 Oct 2013 23:33:45 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix _pow requiring thirdArg on this branch
Message-ID: <20131008213345.60C9F1C026D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67226:127e26c1e91a
Date: 2013-10-08 13:17 -0700
http://bitbucket.org/pypy/pypy/changeset/127e26c1e91a/

Log:	fix _pow requiring thirdArg on this branch

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -398,9 +398,9 @@
     x = w_float1.floatval
     y = w_float2.floatval
 
-    return W_FloatObject(_pow(space, x, y))
+    return W_FloatObject(_pow(space, x, y, thirdArg))
 
-def _pow(space, x, y):
+def _pow(space, x, y, thirdArg):
     # Sort out special cases here instead of relying on pow()
     if y == 2.0:       # special case for performance:
         return x * x   # x * x is always correct

From noreply at buildbot.pypy.org  Wed Oct  9 00:01:28 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed,  9 Oct 2013 00:01:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: Inline into the select module
Message-ID: <20131008220128.DB8361C0161@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67227:19a57b920a2a
Date: 2013-10-08 15:00 -0700
http://bitbucket.org/pypy/pypy/changeset/19a57b920a2a/

Log:	Inline into the select module

diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -110,7 +110,7 @@
                        '__pypy__', 'cStringIO', '_collections', 'struct',
                        'mmap', 'marshal', '_codecs', 'rctime', 'cppyy',
                        '_cffi_backend', 'pyexpat', '_continuation', '_io',
-                       'thread']:
+                       'thread', 'select']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -49,12 +49,15 @@
     from pypy.module.thread.os_lock import Lock
     assert pypypolicy.look_inside_function(Lock.descr_lock_acquire.im_func)
 
+def test_select():
+    from pypy.module.select.interp_select import poll
+    assert pypypolicy.look_inside_function(poll)
+
 def test_pypy_module():
     from pypy.module._collections.interp_deque import W_Deque
     from pypy.module._random.interp_random import W_Random
     assert not pypypolicy.look_inside_function(W_Random.random)
     assert pypypolicy.look_inside_function(W_Deque.length)
-    assert not pypypolicy.look_inside_pypy_module('select.interp_epoll')
     assert pypypolicy.look_inside_pypy_module('__builtin__.operation')
     assert pypypolicy.look_inside_pypy_module('__builtin__.abstractinst')
     assert pypypolicy.look_inside_pypy_module('__builtin__.functional')

From noreply at buildbot.pypy.org  Wed Oct  9 00:40:32 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed,  9 Oct 2013 00:40:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: Change the way event data is
 represented in kqueue. W_Kevent no longer contains a pointer to a struct
 kevent, all the data is now just fields in the W_Kevent object
Message-ID: <20131008224032.5B10B1C0161@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67228:b028e2cbd16b
Date: 2013-10-08 15:39 -0700
http://bitbucket.org/pypy/pypy/changeset/b028e2cbd16b/

Log:	Change the way event data is represented in kqueue. W_Kevent no
	longer contains a pointer to a struct kevent, all the data is now
	just fields in the W_Kevent object

diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -180,12 +180,12 @@
                         i = 0
                         for w_ev in space.listview(w_changelist):
                             ev = space.interp_w(W_Kevent, w_ev)
-                            changelist[i].c_ident = ev.event.c_ident
-                            changelist[i].c_filter = ev.event.c_filter
-                            changelist[i].c_flags = ev.event.c_flags
-                            changelist[i].c_fflags = ev.event.c_fflags
-                            changelist[i].c_data = ev.event.c_data
-                            changelist[i].c_udata = ev.event.c_udata
+                            changelist[i].c_ident = ev.ident
+                            changelist[i].c_filter = ev.filter
+                            changelist[i].c_flags = ev.flags
+                            changelist[i].c_fflags = ev.fflags
+                            changelist[i].c_data = ev.data
+                            changelist[i].c_udata = ev.udata
                             i += 1
                         pchangelist = changelist
                     else:
@@ -206,13 +206,12 @@
                             evt = eventlist[i]
 
                             w_event = W_Kevent(space)
-                            w_event.event = lltype.malloc(kevent, flavor="raw")
-                            w_event.event.c_ident = evt.c_ident
-                            w_event.event.c_filter = evt.c_filter
-                            w_event.event.c_flags = evt.c_flags
-                            w_event.event.c_fflags = evt.c_fflags
-                            w_event.event.c_data = evt.c_data
-                            w_event.event.c_udata = evt.c_udata
+                            w_event.ident = evt.c_ident
+                            w_event.filter = evt.c_filter
+                            w_event.flags = evt.c_flags
+                            w_event.fflags = evt.c_fflags
+                            w_event.data = evt.c_data
+                            w_event.udata = evt.c_udata
 
                             elist_w[i] = w_event
 
@@ -234,11 +233,12 @@
 
 class W_Kevent(W_Root):
     def __init__(self, space):
-        self.event = lltype.nullptr(kevent)
-
-    def __del__(self):
-        if self.event:
-            lltype.free(self.event, flavor="raw")
+        self.ident = rffi.cast(kevent.c_ident, 0)
+        self.filter = rffi.cast(kevent.c_filter, 0)
+        self.flags = rffi.cast(kevent.c_flags, 0)
+        self.fflags = rffi.cast(kevent.c_fflags, 0)
+        self.data = rffi.cast(kevent.c_data, 0)
+        self.udata = lltype.nullptr(rffi.VOIDP.TO)
 
     @unwrap_spec(filter=int, flags='c_uint', fflags='c_uint', data=int, udata=r_uint)
     def descr__init__(self, space, w_ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=r_uint(0)):
@@ -247,35 +247,34 @@
         else:
             ident = r_uint(space.c_filedescriptor_w(w_ident))
 
-        self.event = lltype.malloc(kevent, flavor="raw")
-        rffi.setintfield(self.event, "c_ident", ident)
-        rffi.setintfield(self.event, "c_filter", filter)
-        rffi.setintfield(self.event, "c_flags", flags)
-        rffi.setintfield(self.event, "c_fflags", fflags)
-        rffi.setintfield(self.event, "c_data", data)
-        self.event.c_udata = rffi.cast(rffi.VOIDP, udata)
+        self.ident = rffi.cast(kevent.c_ident, ident)
+        self.filter = rffi.cast(kevent.c_filter, filter)
+        self.flags = rffi.cast(kevent.c_flags, flags)
+        self.fflags = rffi.cast(kevent.c_fflags, fflags)
+        self.data = rffi.cast(kevent.c_data, data)
+        self.udata = rffi.cast(rffi.VOIDP, udata)
 
     def _compare_all_fields(self, other, op):
         if IDENT_UINT:
-            l_ident = rffi.cast(lltype.Unsigned, self.event.c_ident)
-            r_ident = rffi.cast(lltype.Unsigned, other.event.c_ident)
+            l_ident = rffi.cast(lltype.Unsigned, self.ident)
+            r_ident = rffi.cast(lltype.Unsigned, other.ident)
         else:
-            l_ident = self.event.c_ident
-            r_ident = other.event.c_ident
-        l_filter = rffi.cast(lltype.Signed, self.event.c_filter)
-        r_filter = rffi.cast(lltype.Signed, other.event.c_filter)
-        l_flags = rffi.cast(lltype.Unsigned, self.event.c_flags)
-        r_flags = rffi.cast(lltype.Unsigned, other.event.c_flags)
-        l_fflags = rffi.cast(lltype.Unsigned, self.event.c_fflags)
-        r_fflags = rffi.cast(lltype.Unsigned, other.event.c_fflags)
+            l_ident = self.ident
+            r_ident = other.ident
+        l_filter = rffi.cast(lltype.Signed, self.filter)
+        r_filter = rffi.cast(lltype.Signed, other.filter)
+        l_flags = rffi.cast(lltype.Unsigned, self.flags)
+        r_flags = rffi.cast(lltype.Unsigned, other.flags)
+        l_fflags = rffi.cast(lltype.Unsigned, self.fflags)
+        r_fflags = rffi.cast(lltype.Unsigned, other.fflags)
         if IDENT_UINT:
-            l_data = rffi.cast(lltype.Signed, self.event.c_data)
-            r_data = rffi.cast(lltype.Signed, other.event.c_data)
+            l_data = rffi.cast(lltype.Signed, self.data)
+            r_data = rffi.cast(lltype.Signed, other.data)
         else:
-            l_data = self.event.c_data
-            r_data = other.event.c_data
-        l_udata = rffi.cast(lltype.Unsigned, self.event.c_udata)
-        r_udata = rffi.cast(lltype.Unsigned, other.event.c_udata)
+            l_data = self.data
+            r_data = other.data
+        l_udata = rffi.cast(lltype.Unsigned, self.udata)
+        r_udata = rffi.cast(lltype.Unsigned, other.udata)
 
         if op == "eq":
             return l_ident == r_ident and \
@@ -330,22 +329,22 @@
         return space.wrap(self.compare_all_fields(space, w_other, "gt"))
 
     def descr_get_ident(self, space):
-        return space.wrap(self.event.c_ident)
+        return space.wrap(self.ident)
 
     def descr_get_filter(self, space):
-        return space.wrap(self.event.c_filter)
+        return space.wrap(self.filter)
 
     def descr_get_flags(self, space):
-        return space.wrap(self.event.c_flags)
+        return space.wrap(self.flags)
 
     def descr_get_fflags(self, space):
-        return space.wrap(self.event.c_fflags)
+        return space.wrap(self.fflags)
 
     def descr_get_data(self, space):
-        return space.wrap(self.event.c_data)
+        return space.wrap(self.data)
 
     def descr_get_udata(self, space):
-        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.event.c_udata))
+        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.udata))
 
 
 W_Kevent.typedef = TypeDef("select.kevent",

From noreply at buildbot.pypy.org  Wed Oct  9 09:09:53 2013
From: noreply at buildbot.pypy.org (bivab)
Date: Wed,  9 Oct 2013 09:09:53 +0200 (CEST)
Subject: [pypy-commit] buildbot default: add nightly runs of own tests on ARM
Message-ID: <20131009070953.174A81C3208@cobra.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r865:4024f825e023
Date: 2013-10-09 09:08 +0200
http://bitbucket.org/pypy/buildbot/changeset/4024f825e023/

Log:	add nightly runs of own tests on ARM

diff --git a/bot2/pypybuildbot/arm_master.py b/bot2/pypybuildbot/arm_master.py
--- a/bot2/pypybuildbot/arm_master.py
+++ b/bot2/pypybuildbot/arm_master.py
@@ -147,6 +147,8 @@
         BUILDLINUXARM,                 # on hhu-cross-armel, uses 1 core
         BUILDLINUXARMHF_RASPBIAN,      # on hhu-cross-raspbianhf, uses 1 core
 
+        LINUXARMHF,                    # onw tests on greenbox3-node0
+
         JITBACKENDONLYLINUXARMEL,      # on hhu-imx.53
         JITBACKENDONLYLINUXARMHF,
         JITBACKENDONLYLINUXARMHF_v7,   # on cubieboard-bob

From noreply at buildbot.pypy.org  Wed Oct  9 09:57:05 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 09:57:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: be a little more stringent about
 checking negative numbers early
Message-ID: <20131009075705.21AC11C02A3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67229:c214edb11c30
Date: 2013-10-09 09:55 +0200
http://bitbucket.org/pypy/pypy/changeset/c214edb11c30/

Log:	be a little more stringent about checking negative numbers early

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -333,12 +333,13 @@
         check_negative_slice(s_start, s_stop)
         lst.listdef.resize()
 
-def check_negative_slice(s_start, s_stop):
+def check_negative_slice(s_start, s_stop, error="slicing"):
     if isinstance(s_start, SomeInteger) and not s_start.nonneg:
-        raise AnnotatorError("slicing: not proven to have non-negative start")
+        raise AnnotatorError("%s: not proven to have non-negative start" %
+                             error)
     if isinstance(s_stop, SomeInteger) and not s_stop.nonneg and \
            getattr(s_stop, 'const', 0) != -1:
-        raise AnnotatorError("slicing: not proven to have non-negative stop")
+        raise AnnotatorError("%s: not proven to have non-negative stop" % error)
 
 
 class __extend__(SomeDict):
@@ -448,12 +449,15 @@
         return s_Bool
 
     def method_find(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "find")
         return SomeInteger()
 
     def method_rfind(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "rfind")
         return SomeInteger()
 
     def method_count(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "count")
         return SomeInteger(nonneg=True)
 
     def method_strip(str, chr):

From noreply at buildbot.pypy.org  Wed Oct  9 10:52:39 2013
From: noreply at buildbot.pypy.org (zyv)
Date: Wed,  9 Oct 2013 10:52:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix segfaults when encoding is NULL in
 CPyExt Unicode functions
Message-ID: <20131009085239.A0CFA1C0271@cobra.cs.uni-duesseldorf.de>

Author: Yury V. Zaytsev <yury at shurup.com>
Branch: 
Changeset: r67230:3085368d407a
Date: 2013-10-09 10:37 +0200
http://bitbucket.org/pypy/pypy/changeset/3085368d407a/

Log:	Fix segfaults when encoding is NULL in CPyExt Unicode functions

	Functions affected are PyUnicode_SetDefaultEncoding,
	PyUnicode_Decode, PyUnicode_FromEncodedObject. The current behavior
	(whether to raise an exception or to use the default encoding)
	tracks CPython 2.7 semantics.

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -91,6 +91,7 @@
         invalid = rffi.str2charp('invalid')
         utf_8 = rffi.str2charp('utf-8')
         prev_encoding = rffi.str2charp(space.unwrap(w_default_encoding))
+        self.raises(space, api, TypeError, api.PyUnicode_SetDefaultEncoding, lltype.nullptr(rffi.CCHARP.TO))
         assert api.PyUnicode_SetDefaultEncoding(invalid) == -1
         assert api.PyErr_Occurred() is space.w_LookupError
         api.PyErr_Clear()
@@ -316,6 +317,15 @@
         rffi.free_charp(b_text)
         rffi.free_charp(b_encoding)
 
+    def test_decode_null_encoding(self, space, api):
+        null_charp = lltype.nullptr(rffi.CCHARP.TO)
+        u_text = u'abcdefg'
+        s_text = space.str_w(api.PyUnicode_AsEncodedString(space.wrap(u_text), null_charp, null_charp))
+        b_text = rffi.str2charp(s_text)
+        assert space.unwrap(api.PyUnicode_Decode(b_text, len(s_text), null_charp, null_charp)) == u_text
+        self.raises(space, api, TypeError, api.PyUnicode_FromEncodedObject, space.wrap(u_text), null_charp, None)
+        rffi.free_charp(b_text)
+
     def test_leak(self):
         size = 50
         raw_buf, gc_buf = rffi.alloc_buffer(size)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -273,6 +273,8 @@
 def PyUnicode_SetDefaultEncoding(space, encoding):
     """Sets the currently active default encoding. Returns 0 on
     success, -1 in case of an error."""
+    if not encoding:
+        PyErr_BadArgument(space)
     w_encoding = space.wrap(rffi.charp2str(encoding))
     setdefaultencoding(space, w_encoding)
     default_encoding[0] = '\x00'
@@ -350,8 +352,11 @@
     in the unicode() built-in function.  The codec to be used is looked up
     using the Python codec registry.  Return NULL if an exception was raised by
     the codec."""
+    if not encoding:
+        # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead
+        encoding = PyUnicode_GetDefaultEncoding(space)
+    w_encoding = space.wrap(rffi.charp2str(encoding))
     w_str = space.wrap(rffi.charpsize2str(s, size))
-    w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))
     else:
@@ -379,6 +384,9 @@
 
     All other objects, including Unicode objects, cause a TypeError to be
     set."""
+    if not encoding:
+        raise OperationError(space.w_TypeError,
+                             space.wrap("decoding Unicode is not supported"))
     w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))

From noreply at buildbot.pypy.org  Wed Oct  9 10:52:41 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 10:52:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged in zyv/pypy (pull request #194)
Message-ID: <20131009085241.36BC21C0271@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67231:798905aca6f1
Date: 2013-10-09 10:51 +0200
http://bitbucket.org/pypy/pypy/changeset/798905aca6f1/

Log:	Merged in zyv/pypy (pull request #194)

	Fix segfaults when encoding is NULL in CPyExt Unicode functions

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -91,6 +91,7 @@
         invalid = rffi.str2charp('invalid')
         utf_8 = rffi.str2charp('utf-8')
         prev_encoding = rffi.str2charp(space.unwrap(w_default_encoding))
+        self.raises(space, api, TypeError, api.PyUnicode_SetDefaultEncoding, lltype.nullptr(rffi.CCHARP.TO))
         assert api.PyUnicode_SetDefaultEncoding(invalid) == -1
         assert api.PyErr_Occurred() is space.w_LookupError
         api.PyErr_Clear()
@@ -316,6 +317,15 @@
         rffi.free_charp(b_text)
         rffi.free_charp(b_encoding)
 
+    def test_decode_null_encoding(self, space, api):
+        null_charp = lltype.nullptr(rffi.CCHARP.TO)
+        u_text = u'abcdefg'
+        s_text = space.str_w(api.PyUnicode_AsEncodedString(space.wrap(u_text), null_charp, null_charp))
+        b_text = rffi.str2charp(s_text)
+        assert space.unwrap(api.PyUnicode_Decode(b_text, len(s_text), null_charp, null_charp)) == u_text
+        self.raises(space, api, TypeError, api.PyUnicode_FromEncodedObject, space.wrap(u_text), null_charp, None)
+        rffi.free_charp(b_text)
+
     def test_leak(self):
         size = 50
         raw_buf, gc_buf = rffi.alloc_buffer(size)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -273,6 +273,8 @@
 def PyUnicode_SetDefaultEncoding(space, encoding):
     """Sets the currently active default encoding. Returns 0 on
     success, -1 in case of an error."""
+    if not encoding:
+        PyErr_BadArgument(space)
     w_encoding = space.wrap(rffi.charp2str(encoding))
     setdefaultencoding(space, w_encoding)
     default_encoding[0] = '\x00'
@@ -350,8 +352,11 @@
     in the unicode() built-in function.  The codec to be used is looked up
     using the Python codec registry.  Return NULL if an exception was raised by
     the codec."""
+    if not encoding:
+        # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead
+        encoding = PyUnicode_GetDefaultEncoding(space)
+    w_encoding = space.wrap(rffi.charp2str(encoding))
     w_str = space.wrap(rffi.charpsize2str(s, size))
-    w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))
     else:
@@ -379,6 +384,9 @@
 
     All other objects, including Unicode objects, cause a TypeError to be
     set."""
+    if not encoding:
+        raise OperationError(space.w_TypeError,
+                             space.wrap("decoding Unicode is not supported"))
     w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))

From noreply at buildbot.pypy.org  Wed Oct  9 11:26:52 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 11:26:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: implement os.ctermid
Message-ID: <20131009092652.5B1C61C02A3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67232:425cfc2cbfb2
Date: 2013-10-09 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/425cfc2cbfb2/

Log:	implement os.ctermid

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -50,6 +50,7 @@
         'lstat': 'interp_posix.lstat',
         'stat_float_times': 'interp_posix.stat_float_times',
 
+        'ctermid': 'interp_posix.ctermid',
         'dup': 'interp_posix.dup',
         'dup2': 'interp_posix.dup2',
         'access': 'interp_posix.access',
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1218,3 +1218,10 @@
         return space.wrap(rurandom.urandom(context, n))
     except OSError, e:
         raise wrap_oserror(space, e)
+
+def ctermid(space):
+    """ctermid() -> string
+
+    Return the name of the controlling terminal for this process.
+    """
+    return space.wrap(os.ctermid())

From noreply at buildbot.pypy.org  Wed Oct  9 13:55:42 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  9 Oct 2013 13:55:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: also support intlist[a:b:c] = rangelist
	(useful!)
Message-ID: <20131009115542.F200B1C3230@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r67234:b1f42572695e
Date: 2013-10-09 12:21 +0200
http://bitbucket.org/pypy/pypy/changeset/b1f42572695e/

Log:	also support intlist[a:b:c] = rangelist (useful!)

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1328,7 +1328,6 @@
 
     def setslice(self, w_list, start, step, slicelength, w_other):
         assert slicelength >= 0
-        items = self.unerase(w_list.lstorage)
 
         if self is self.space.fromcache(ObjectListStrategy):
             w_other = w_other._temporarily_as_objects()
@@ -1340,6 +1339,7 @@
             w_list.setslice(start, step, slicelength, w_other_as_object)
             return
 
+        items = self.unerase(w_list.lstorage)
         oldsize = len(items)
         len2 = w_other.length()
         if step == 1:  # Support list resizing for non-extended slices
@@ -1535,6 +1535,15 @@
         return self._base_extend_from_list(w_list, w_other)
 
 
+    _base_setslice = setslice
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            storage = self.erase(w_other.getitems_int())
+            w_other = W_ListObject.from_storage_and_strategy(
+                    self.space, storage, self)
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
 class FloatListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -225,6 +225,15 @@
         l.setslice(0, 1, 2, W_ListObject(space, [w('a'), w(2), w(3)]))
         assert isinstance(l.strategy, ObjectListStrategy)
 
+    def test_setslice_int_range(self):
+        space = self.space
+        w = space.wrap
+        l = W_ListObject(space, [w(1), w(2), w(3)])
+        assert isinstance(l.strategy, IntegerListStrategy)
+        l.setslice(0, 1, 2, make_range_list(space, 5, 1, 4))
+        assert isinstance(l.strategy, IntegerListStrategy)
+
+
     def test_setslice_List(self):
         space = self.space
 

From noreply at buildbot.pypy.org  Wed Oct  9 13:55:40 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  9 Oct 2013 13:55:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: support adding a range list to an int
 list to get an int list
Message-ID: <20131009115540.036DF1C15BB@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r67233:e47f03fa8558
Date: 2013-10-09 12:14 +0200
http://bitbucket.org/pypy/pypy/changeset/e47f03fa8558/

Log:	support adding a range list to an int list to get an int list

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -30,7 +30,7 @@
 from rpython.rlib import debug, jit, rerased
 from rpython.rlib.listsort import make_timsort_class
 from rpython.rlib.objectmodel import (
-    instantiate, newlist_hint, resizelist_hint, specialize)
+    instantiate, newlist_hint, resizelist_hint, specialize, import_from_mixin)
 from rpython.tool.sourcetools import func_with_new_name
 
 __all__ = ['W_ListObject', 'make_range_list', 'make_empty_list_with_size']
@@ -1170,7 +1170,6 @@
 
 
 class AbstractUnwrappedStrategy(object):
-    _mixin_ = True
 
     def wrap(self, unwrapped):
         raise NotImplementedError
@@ -1456,7 +1455,9 @@
         self.unerase(w_list.lstorage).reverse()
 
 
-class ObjectListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class ObjectListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "object"
 
@@ -1489,7 +1490,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class IntegerListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class IntegerListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0
     _applevel_repr = "int"
 
@@ -1520,7 +1523,21 @@
         return self.unerase(w_list.lstorage)
 
 
-class FloatListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            l = self.unerase(w_list.lstorage)
+            other = w_other.getitems_int()
+            assert other is not None
+            l += other
+            return
+        return self._base_extend_from_list(w_list, w_other)
+
+
+class FloatListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0.0
     _applevel_repr = "float"
 
@@ -1548,7 +1565,9 @@
             l.reverse()
 
 
-class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class StringListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "str"
 
@@ -1579,7 +1598,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class UnicodeListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class UnicodeListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "unicode"
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -467,6 +467,12 @@
         l4 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3), self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         assert self.space.eq_w(l3, l4)
 
+    def test_add_of_range_and_int(self):
+        l1 = make_range_list(self.space, 0, 1, 100)
+        l2 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
+        l3 = self.space.add(l2, l1)
+        assert l3.strategy is l2.strategy
+
     def test_mul(self):
         l1 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         l2 = l1.mul(2)

From noreply at buildbot.pypy.org  Wed Oct  9 15:35:32 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  9 Oct 2013 15:35:32 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: a branch where to add a
 fastpath for things like ffi.new('long[]', [1, 2, 3]),
 exploiting list strategies
Message-ID: <20131009133532.1C6831C33EE@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67235:32068d2c44b3
Date: 2013-10-08 18:39 +0200
http://bitbucket.org/pypy/pypy/changeset/32068d2c44b3/

Log:	a branch where to add a fastpath for things like ffi.new('long[]',
	[1,2,3]), exploiting list strategies


From noreply at buildbot.pypy.org  Wed Oct  9 15:35:33 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  9 Oct 2013 15:35:33 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: add a function to memcpy
 the content of an rpython list into a raw array
Message-ID: <20131009133533.699881C3597@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67236:8a818cbdcdc0
Date: 2013-10-09 10:48 +0200
http://bitbucket.org/pypy/pypy/changeset/8a818cbdcdc0/

Log:	add a function to memcpy the content of an rpython list into a raw
	array

diff --git a/rpython/rlib/rarray.py b/rpython/rlib/rarray.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rarray.py
@@ -0,0 +1,31 @@
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.extregistry import ExtRegistryEntry
+
+def copy_list_to_raw_array(lst, array):
+    for i, item in enumerate(lst):
+        array[i] = item
+
+        
+class Entry(ExtRegistryEntry):
+    _about_ = copy_list_to_raw_array
+
+    def compute_result_annotation(self, *s_args):
+        pass
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        v_list, v_buf = hop.inputargs(*hop.args_r)
+        return hop.gendirectcall(ll_copy_list_to_raw_array, v_list, v_buf)
+
+
+def ll_copy_list_to_raw_array(ll_list, dst_ptr):
+    src_ptr = ll_list.ll_items()
+    src_adr = llmemory.cast_ptr_to_adr(src_ptr)
+    src_adr += llmemory.itemoffsetof(lltype.typeOf(src_ptr).TO, 0) # skip the GC header
+    #
+    dst_adr = llmemory.cast_ptr_to_adr(dst_ptr)
+    dst_adr += llmemory.itemoffsetof(lltype.typeOf(dst_ptr).TO, 0)
+    #
+    ITEM = lltype.typeOf(dst_ptr).TO.OF
+    size = llmemory.sizeof(ITEM) * ll_list.ll_length()
+    llmemory.raw_memcopy(src_adr, dst_adr, size)
diff --git a/rpython/rlib/test/test_rarray.py b/rpython/rlib/test/test_rarray.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rarray.py
@@ -0,0 +1,38 @@
+from rpython.rlib.rarray import copy_list_to_raw_array
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.test.tool import BaseRtypingTest
+
+
+
+class TestRArray(BaseRtypingTest):
+
+    def test_copy_list_to_raw_array(self):
+        ARRAY = rffi.CArray(lltype.Signed)
+        buf = lltype.malloc(ARRAY, 4, flavor='raw')
+        lst = [1, 2, 3, 4]
+        copy_list_to_raw_array(lst, buf)
+        for i in range(4):
+            assert buf[i] == i+1
+        lltype.free(buf, flavor='raw')
+        
+
+    def test_copy_list_to_raw_array_rtyped(self):
+        INTARRAY = rffi.CArray(lltype.Signed)
+        FLOATARRAY = rffi.CArray(lltype.Float)
+        def fn():
+            buf = lltype.malloc(INTARRAY, 3, flavor='raw')
+            lst = [1, 2, 3]
+            copy_list_to_raw_array(lst, buf)
+            for i in range(3):
+                assert buf[i] == lst[i]
+            #
+            buf2 = lltype.malloc(FLOATARRAY, 3, flavor='raw')
+            lst = [1.1, 2.2, 3.3]
+            copy_list_to_raw_array(lst, buf2)
+            for i in range(3):
+                assert buf2[i] == lst[i]
+            #
+            lltype.free(buf, flavor='raw')
+            lltype.free(buf2, flavor='raw')
+        self.interpret(fn, [])
+    

From noreply at buildbot.pypy.org  Wed Oct  9 15:35:39 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  9 Oct 2013 15:35:39 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: use the official way to
	resize the list
Message-ID: <20131009133539.8153D1C33EE@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67238:03367c66416e
Date: 2013-10-09 15:20 +0200
http://bitbucket.org/pypy/pypy/changeset/03367c66416e/

Log:	use the official way to resize the list

diff --git a/rpython/rlib/rarray.py b/rpython/rlib/rarray.py
--- a/rpython/rlib/rarray.py
+++ b/rpython/rlib/rarray.py
@@ -58,19 +58,16 @@
 
 
 def ll_populate_list_from_raw_array(ll_list, src_ptr, length):
-    PTR_ITEMS = lltype.typeOf(ll_list.items)
-    new_items = lltype.malloc(PTR_ITEMS.TO, length)
+    ll_list._ll_resize(length)
+    ll_items = ll_list.ll_items()
     #
     # start of no-GC section
     src_adr = llmemory.cast_ptr_to_adr(src_ptr)
     src_adr += llmemory.itemoffsetof(lltype.typeOf(src_ptr).TO, 0)
-    dst_adr = llmemory.cast_ptr_to_adr(new_items)
-    dst_adr += llmemory.itemoffsetof(lltype.typeOf(new_items).TO, 0) # skip the GC header
+    dst_adr = llmemory.cast_ptr_to_adr(ll_items)
+    dst_adr += llmemory.itemoffsetof(lltype.typeOf(ll_items).TO, 0) # skip the GC header
     #
     ITEM = lltype.typeOf(src_ptr).TO.OF
     size = llmemory.sizeof(ITEM) * length
     llmemory.raw_memcopy(src_adr, dst_adr, size)
     # end of no-GC section
-    #
-    ll_list.items = new_items
-    ll_list.length = length

From noreply at buildbot.pypy.org  Wed Oct  9 15:35:34 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  9 Oct 2013 15:35:34 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: add a function to memcpy
 the content of a raw array into an rpython list
Message-ID: <20131009133534.8B42C1C359C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67237:2bbec1d6d9cc
Date: 2013-10-09 15:14 +0200
http://bitbucket.org/pypy/pypy/changeset/2bbec1d6d9cc/

Log:	add a function to memcpy the content of a raw array into an rpython
	list

diff --git a/rpython/rlib/rarray.py b/rpython/rlib/rarray.py
--- a/rpython/rlib/rarray.py
+++ b/rpython/rlib/rarray.py
@@ -1,5 +1,8 @@
+from rpython.annotator import model as annmodel
+from rpython.annotator.listdef import ListDef
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.tool.pairtype import pair
 
 def copy_list_to_raw_array(lst, array):
     for i, item in enumerate(lst):
@@ -19,6 +22,10 @@
 
 
 def ll_copy_list_to_raw_array(ll_list, dst_ptr):
+    # this code is delicate: we must ensure that there are no GC operations
+    # between here and the call to raw_memcopy
+    #
+    # start of no-GC section
     src_ptr = ll_list.ll_items()
     src_adr = llmemory.cast_ptr_to_adr(src_ptr)
     src_adr += llmemory.itemoffsetof(lltype.typeOf(src_ptr).TO, 0) # skip the GC header
@@ -29,3 +36,41 @@
     ITEM = lltype.typeOf(dst_ptr).TO.OF
     size = llmemory.sizeof(ITEM) * ll_list.ll_length()
     llmemory.raw_memcopy(src_adr, dst_adr, size)
+    # end of no-GC section
+
+
+def populate_list_from_raw_array(lst, array, length):
+    lst[:] = [array[i] for i in range(length)]
+
+class Entry(ExtRegistryEntry):
+    _about_ = populate_list_from_raw_array
+
+    def compute_result_annotation(self, s_list, s_array, s_length):
+        s_item = annmodel.lltype_to_annotation(s_array.ll_ptrtype.TO.OF)
+        s_newlist = self.bookkeeper.newlist(s_item)
+        s_newlist.listdef.resize()
+        pair(s_list, s_newlist).union()
+
+    def specialize_call(self, hop):
+        v_list, v_buf, v_length = hop.inputargs(*hop.args_r)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_populate_list_from_raw_array, v_list, v_buf, v_length)
+
+
+def ll_populate_list_from_raw_array(ll_list, src_ptr, length):
+    PTR_ITEMS = lltype.typeOf(ll_list.items)
+    new_items = lltype.malloc(PTR_ITEMS.TO, length)
+    #
+    # start of no-GC section
+    src_adr = llmemory.cast_ptr_to_adr(src_ptr)
+    src_adr += llmemory.itemoffsetof(lltype.typeOf(src_ptr).TO, 0)
+    dst_adr = llmemory.cast_ptr_to_adr(new_items)
+    dst_adr += llmemory.itemoffsetof(lltype.typeOf(new_items).TO, 0) # skip the GC header
+    #
+    ITEM = lltype.typeOf(src_ptr).TO.OF
+    size = llmemory.sizeof(ITEM) * length
+    llmemory.raw_memcopy(src_adr, dst_adr, size)
+    # end of no-GC section
+    #
+    ll_list.items = new_items
+    ll_list.length = length
diff --git a/rpython/rlib/test/test_rarray.py b/rpython/rlib/test/test_rarray.py
--- a/rpython/rlib/test/test_rarray.py
+++ b/rpython/rlib/test/test_rarray.py
@@ -1,4 +1,4 @@
-from rpython.rlib.rarray import copy_list_to_raw_array
+from rpython.rlib.rarray import copy_list_to_raw_array, populate_list_from_raw_array
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.test.tool import BaseRtypingTest
 
@@ -35,4 +35,30 @@
             lltype.free(buf, flavor='raw')
             lltype.free(buf2, flavor='raw')
         self.interpret(fn, [])
-    
+
+    def test_new_list_from_raw_array(self):
+        INTARRAY = rffi.CArray(lltype.Signed)
+        buf = lltype.malloc(INTARRAY, 4, flavor='raw')
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        buf[3] = 4
+        lst = []
+        populate_list_from_raw_array(lst, buf, 4)
+        assert lst == [1, 2, 3, 4]
+        lltype.free(buf, flavor='raw')
+
+    def test_new_list_from_raw_array_rtyped(self):
+        INTARRAY = rffi.CArray(lltype.Signed)
+        def fn():
+            buf = lltype.malloc(INTARRAY, 4, flavor='raw')
+            buf[0] = 1
+            buf[1] = 2
+            buf[2] = 3
+            buf[3] = 4
+            lst = []
+            populate_list_from_raw_array(lst, buf, 4)
+            assert lst == [1, 2, 3, 4]
+            lltype.free(buf, flavor='raw')
+        #
+        self.interpret(fn, [])

From noreply at buildbot.pypy.org  Wed Oct  9 15:35:40 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  9 Oct 2013 15:35:40 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: shuffle things around and
 kill a bit of code duplication
Message-ID: <20131009133540.CF66D1C33EE@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67239:0cdf4e71d831
Date: 2013-10-09 15:34 +0200
http://bitbucket.org/pypy/pypy/changeset/0cdf4e71d831/

Log:	shuffle things around and kill a bit of code duplication

diff --git a/rpython/rlib/rarray.py b/rpython/rlib/rarray.py
--- a/rpython/rlib/rarray.py
+++ b/rpython/rlib/rarray.py
@@ -1,5 +1,6 @@
 from rpython.annotator import model as annmodel
 from rpython.annotator.listdef import ListDef
+from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.pairtype import pair
@@ -8,7 +9,11 @@
     for i, item in enumerate(lst):
         array[i] = item
 
-        
+def populate_list_from_raw_array(lst, array, length):
+    lst[:] = [array[i] for i in range(length)]
+
+
+
 class Entry(ExtRegistryEntry):
     _about_ = copy_list_to_raw_array
 
@@ -21,27 +26,6 @@
         return hop.gendirectcall(ll_copy_list_to_raw_array, v_list, v_buf)
 
 
-def ll_copy_list_to_raw_array(ll_list, dst_ptr):
-    # this code is delicate: we must ensure that there are no GC operations
-    # between here and the call to raw_memcopy
-    #
-    # start of no-GC section
-    src_ptr = ll_list.ll_items()
-    src_adr = llmemory.cast_ptr_to_adr(src_ptr)
-    src_adr += llmemory.itemoffsetof(lltype.typeOf(src_ptr).TO, 0) # skip the GC header
-    #
-    dst_adr = llmemory.cast_ptr_to_adr(dst_ptr)
-    dst_adr += llmemory.itemoffsetof(lltype.typeOf(dst_ptr).TO, 0)
-    #
-    ITEM = lltype.typeOf(dst_ptr).TO.OF
-    size = llmemory.sizeof(ITEM) * ll_list.ll_length()
-    llmemory.raw_memcopy(src_adr, dst_adr, size)
-    # end of no-GC section
-
-
-def populate_list_from_raw_array(lst, array, length):
-    lst[:] = [array[i] for i in range(length)]
-
 class Entry(ExtRegistryEntry):
     _about_ = populate_list_from_raw_array
 
@@ -57,17 +41,32 @@
         return hop.gendirectcall(ll_populate_list_from_raw_array, v_list, v_buf, v_length)
 
 
+ at specialize.ll()
+def get_raw_buf(ptr):
+    ofs = llmemory.itemoffsetof(lltype.typeOf(ptr).TO, 0)
+    return llmemory.cast_ptr_to_adr(ptr) + ofs
+get_raw_buf._always_inline_ = True
+
+
+def ll_copy_list_to_raw_array(ll_list, dst_ptr):
+    # this code is delicate: we must ensure that there are no GC operations
+    # around the call to raw_memcopy
+    #
+    ITEM = lltype.typeOf(dst_ptr).TO.OF
+    size = llmemory.sizeof(ITEM) * ll_list.ll_length()
+    # start of no-GC section
+    src_adr = get_raw_buf(ll_list.ll_items())
+    dst_adr = get_raw_buf(dst_ptr)
+    llmemory.raw_memcopy(src_adr, dst_adr, size)
+    # end of no-GC section
+
+
 def ll_populate_list_from_raw_array(ll_list, src_ptr, length):
-    ll_list._ll_resize(length)
-    ll_items = ll_list.ll_items()
-    #
-    # start of no-GC section
-    src_adr = llmemory.cast_ptr_to_adr(src_ptr)
-    src_adr += llmemory.itemoffsetof(lltype.typeOf(src_ptr).TO, 0)
-    dst_adr = llmemory.cast_ptr_to_adr(ll_items)
-    dst_adr += llmemory.itemoffsetof(lltype.typeOf(ll_items).TO, 0) # skip the GC header
-    #
     ITEM = lltype.typeOf(src_ptr).TO.OF
     size = llmemory.sizeof(ITEM) * length
+    ll_list._ll_resize(length)
+    # start of no-GC section
+    src_adr = get_raw_buf(src_ptr)
+    dst_adr = get_raw_buf(ll_list.ll_items())
     llmemory.raw_memcopy(src_adr, dst_adr, size)
     # end of no-GC section

From noreply at buildbot.pypy.org  Wed Oct  9 16:00:56 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 16:00:56 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-2: a branch to restart
Message-ID: <20131009140056.C37B11C3597@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-2
Changeset: r67240:31b7c9223f79
Date: 2013-10-09 11:53 +0200
http://bitbucket.org/pypy/pypy/changeset/31b7c9223f79/

Log:	a branch to restart


From noreply at buildbot.pypy.org  Wed Oct  9 16:00:58 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 16:00:58 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal,
 arigo) Create a branch for reviving rdict experiments
Message-ID: <20131009140058.2FBB81C3597@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67241:df6c605a7c38
Date: 2013-10-09 11:54 +0200
http://bitbucket.org/pypy/pypy/changeset/df6c605a7c38/

Log:	(fijal, arigo) Create a branch for reviving rdict experiments


From noreply at buildbot.pypy.org  Wed Oct  9 16:00:59 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 16:00:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: unbreak the translation on top of pypy
Message-ID: <20131009140059.5122F1C3597@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67242:054c72445b71
Date: 2013-10-09 15:02 +0200
http://bitbucket.org/pypy/pypy/changeset/054c72445b71/

Log:	unbreak the translation on top of pypy

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -50,7 +50,6 @@
         'lstat': 'interp_posix.lstat',
         'stat_float_times': 'interp_posix.stat_float_times',
 
-        'ctermid': 'interp_posix.ctermid',
         'dup': 'interp_posix.dup',
         'dup2': 'interp_posix.dup2',
         'access': 'interp_posix.access',
@@ -150,6 +149,8 @@
         interpleveldefs['nice'] = 'interp_posix.nice'
     if hasattr(os, 'getlogin'):
         interpleveldefs['getlogin'] = 'interp_posix.getlogin'
+    if hasattr(os, 'ctermid'):
+        interpleveldefs['ctermid'] = 'interp_posix.ctermid'
 
     for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid',
                  'seteuid', 'setgid', 'setegid', 'getgroups', 'getpgrp',

From noreply at buildbot.pypy.org  Wed Oct  9 16:01:00 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 16:01:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131009140100.A582A1C3597@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67243:01608017108a
Date: 2013-10-09 15:03 +0200
http://bitbucket.org/pypy/pypy/changeset/01608017108a/

Log:	merge

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -30,7 +30,7 @@
 from rpython.rlib import debug, jit, rerased
 from rpython.rlib.listsort import make_timsort_class
 from rpython.rlib.objectmodel import (
-    instantiate, newlist_hint, resizelist_hint, specialize)
+    instantiate, newlist_hint, resizelist_hint, specialize, import_from_mixin)
 from rpython.tool.sourcetools import func_with_new_name
 
 __all__ = ['W_ListObject', 'make_range_list', 'make_empty_list_with_size']
@@ -1170,7 +1170,6 @@
 
 
 class AbstractUnwrappedStrategy(object):
-    _mixin_ = True
 
     def wrap(self, unwrapped):
         raise NotImplementedError
@@ -1329,7 +1328,6 @@
 
     def setslice(self, w_list, start, step, slicelength, w_other):
         assert slicelength >= 0
-        items = self.unerase(w_list.lstorage)
 
         if self is self.space.fromcache(ObjectListStrategy):
             w_other = w_other._temporarily_as_objects()
@@ -1341,6 +1339,7 @@
             w_list.setslice(start, step, slicelength, w_other_as_object)
             return
 
+        items = self.unerase(w_list.lstorage)
         oldsize = len(items)
         len2 = w_other.length()
         if step == 1:  # Support list resizing for non-extended slices
@@ -1456,7 +1455,9 @@
         self.unerase(w_list.lstorage).reverse()
 
 
-class ObjectListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class ObjectListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "object"
 
@@ -1489,7 +1490,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class IntegerListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class IntegerListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0
     _applevel_repr = "int"
 
@@ -1520,7 +1523,30 @@
         return self.unerase(w_list.lstorage)
 
 
-class FloatListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            l = self.unerase(w_list.lstorage)
+            other = w_other.getitems_int()
+            assert other is not None
+            l += other
+            return
+        return self._base_extend_from_list(w_list, w_other)
+
+
+    _base_setslice = setslice
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            storage = self.erase(w_other.getitems_int())
+            w_other = W_ListObject.from_storage_and_strategy(
+                    self.space, storage, self)
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
+class FloatListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0.0
     _applevel_repr = "float"
 
@@ -1548,7 +1574,9 @@
             l.reverse()
 
 
-class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class StringListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "str"
 
@@ -1579,7 +1607,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class UnicodeListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class UnicodeListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "unicode"
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -225,6 +225,15 @@
         l.setslice(0, 1, 2, W_ListObject(space, [w('a'), w(2), w(3)]))
         assert isinstance(l.strategy, ObjectListStrategy)
 
+    def test_setslice_int_range(self):
+        space = self.space
+        w = space.wrap
+        l = W_ListObject(space, [w(1), w(2), w(3)])
+        assert isinstance(l.strategy, IntegerListStrategy)
+        l.setslice(0, 1, 2, make_range_list(space, 5, 1, 4))
+        assert isinstance(l.strategy, IntegerListStrategy)
+
+
     def test_setslice_List(self):
         space = self.space
 
@@ -467,6 +476,12 @@
         l4 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3), self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         assert self.space.eq_w(l3, l4)
 
+    def test_add_of_range_and_int(self):
+        l1 = make_range_list(self.space, 0, 1, 100)
+        l2 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
+        l3 = self.space.add(l2, l1)
+        assert l3.strategy is l2.strategy
+
     def test_mul(self):
         l1 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         l2 = l1.mul(2)

From noreply at buildbot.pypy.org  Wed Oct  9 16:01:02 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 16:01:02 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal,
 arigo) whack whack whack until we make the first test pass
Message-ID: <20131009140102.4031F1C3597@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67244:1e3bab885783
Date: 2013-10-09 15:46 +0200
http://bitbucket.org/pypy/pypy/changeset/1e3bab885783/

Log:	(fijal, arigo) whack whack whack until we make the first test pass

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -1,43 +1,153 @@
+import sys
 from rpython.tool.pairtype import pairtype
 from rpython.flowspace.model import Constant
 from rpython.rtyper.rdict import AbstractDictRepr, AbstractDictIteratorRepr
-from rpython.rtyper.lltypesystem import lltype
-from rpython.rlib import objectmodel, jit
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rlib import objectmodel, jit, rgc
 from rpython.rlib.debug import ll_assert
-from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT
+from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rtyper import rmodel
 from rpython.rtyper.error import TyperError
+from rpython.rtyper.annlowlevel import llhelper
 
 
-HIGHEST_BIT = r_uint(intmask(1 << (LONG_BIT - 1)))
-MASK = r_uint(intmask(HIGHEST_BIT - 1))
-
 # ____________________________________________________________
 #
 #  generic implementation of RPython dictionary, with parametric DICTKEY and
-#  DICTVALUE types.
+#  DICTVALUE types. The basic implementation is a sparse array of indexes
+#  plus a dense array of structs that contain keys and values. struct looks
+#  like that:
 #
-#  XXX for immutable dicts, the array should be inlined and
-#      resize_counter and everused are not needed.
 #
 #    struct dictentry {
 #        DICTKEY key;
+#        DICTVALUE value;
+#        long f_hash;        # (optional) key hash, if hard to recompute
 #        bool f_valid;      # (optional) the entry is filled
-#        bool f_everused;   # (optional) the entry is or has ever been filled
-#        DICTVALUE value;
-#        int f_hash;        # (optional) key hash, if hard to recompute
 #    }
 #
 #    struct dicttable {
 #        int num_items;
+#        int num_used_items;
 #        int resize_counter;
-#        Array *entries;
+#        {byte, short, int, long} *indexes;
+#        dictentry *entries;
+#        lookup_function; # one of the four possible functions for different
+#                         # size dicts
 #        (Function DICTKEY, DICTKEY -> bool) *fnkeyeq;
 #        (Function DICTKEY -> int) *fnkeyhash;
 #    }
 #
 #
 
+def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
+                ll_fasthash_function=None, ll_hash_function=None,
+                ll_eq_function=None, method_cache={},
+                dummykeyobj=None, dummyvalueobj=None):
+    # get the actual DICT type. if DICT is None, it's created, otherwise
+    # forward reference is becoming DICT
+    if DICT is None:
+        DICT = lltype.GcForwardReference()
+    # compute the shape of the DICTENTRY structure
+    entryfields = []
+    entrymeths = {
+        'allocate': lltype.typeMethod(_ll_malloc_entries),
+        'delete': _ll_free_entries,
+        'must_clear_key':   (isinstance(DICTKEY, lltype.Ptr)
+                             and DICTKEY._needsgc()),
+        'must_clear_value': (isinstance(DICTVALUE, lltype.Ptr)
+                             and DICTVALUE._needsgc()),
+        }
+
+    # * the key
+    entryfields.append(("key", DICTKEY))
+
+    # * the state of the entry - trying to encode it as dummy objects
+    if dummykeyobj:
+        # all the state can be encoded in the key
+        entrymeths['dummy_obj'] = dummykeyobj
+        entrymeths['valid'] = ll_valid_from_key
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_key
+        # the key is overwritten by 'dummy' when the entry is deleted
+        entrymeths['must_clear_key'] = False
+
+    elif dummyvalueobj:
+        # all the state can be encoded in the value
+        entrymeths['dummy_obj'] = dummyvalueobj
+        entrymeths['valid'] = ll_valid_from_value
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_value
+        # value is overwritten by 'dummy' when entry is deleted
+        entrymeths['must_clear_value'] = False
+
+    else:
+        # we need a flag to know if the entry was ever used
+        entryfields.append(("f_valid", lltype.Bool))
+        entrymeths['valid'] = ll_valid_from_flag
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_flag
+
+    # * the value
+    entryfields.append(("value", DICTVALUE))
+
+    if ll_fasthash_function is None:
+        entryfields.append(("f_hash", lltype.Signed))
+        entrymeths['hash'] = ll_hash_from_cache
+    else:
+        entrymeths['hash'] = ll_hash_recomputed
+        entrymeths['fasthashfn'] = ll_fasthash_function
+
+    # Build the lltype data structures
+    DICTENTRY = lltype.Struct("dictentry", *entryfields)
+    DICTENTRYARRAY = lltype.GcArray(DICTENTRY,
+                                    adtmeths=entrymeths)
+    LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY, lltype.Signed, lltype.Signed], lltype.Signed))
+
+
+    fields =          [ ("num_items", lltype.Signed),
+                        ("num_used_items", lltype.Signed),
+                        ("resize_counter", lltype.Signed),
+                        ("indexes", llmemory.GCREF),
+                        ("lookup_function", LOOKUP_FUNC),
+                        ("entries", lltype.Ptr(DICTENTRYARRAY)) ]
+    if get_custom_eq_hash is not None:
+        r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash()
+        fields.extend([ ("fnkeyeq", r_rdict_eqfn.lowleveltype),
+                        ("fnkeyhash", r_rdict_hashfn.lowleveltype) ])
+        adtmeths = {
+            'keyhash':        ll_keyhash_custom,
+            'keyeq':          ll_keyeq_custom,
+            'r_rdict_eqfn':   r_rdict_eqfn,
+            'r_rdict_hashfn': r_rdict_hashfn,
+            'paranoia':       True,
+            }
+    else:
+        # figure out which functions must be used to hash and compare
+        ll_keyhash = ll_hash_function
+        ll_keyeq = ll_eq_function
+        ll_keyhash = lltype.staticAdtMethod(ll_keyhash)
+        if ll_keyeq is not None:
+            ll_keyeq = lltype.staticAdtMethod(ll_keyeq)
+        adtmeths = {
+            'keyhash':  ll_keyhash,
+            'keyeq':    ll_keyeq,
+            'paranoia': False,
+            }
+    adtmeths['KEY']   = DICTKEY
+    adtmeths['VALUE'] = DICTVALUE
+    adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
+    adtmeths['empty_array'] = DICTENTRYARRAY.allocate(0)
+    adtmeths['byte_lookup_function'] = new_lookup_function(LOOKUP_FUNC,
+                                                           T=rffi.UCHAR)
+    adtmeths['short_lookup_function'] = new_lookup_function(LOOKUP_FUNC,
+                                                            T=rffi.USHORT)
+    if IS_64BIT:
+        adtmeths['int_lookup_function'] = new_lookup_function(LOOKUP_FUNC,
+                                                              T=rffi.UINT)
+    adtmeths['long_lookup_function'] = new_lookup_function(LOOKUP_FUNC,
+                                                           T=lltype.Unsigned)
+    DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
+                                *fields))
+    return DICT
+
 class DictRepr(AbstractDictRepr):
 
     def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
@@ -73,140 +183,29 @@
         if 'value_repr' not in self.__dict__:
             self.external_value_repr, self.value_repr = self.pickrepr(self._value_repr_computer())
         if isinstance(self.DICT, lltype.GcForwardReference):
-            self.DICTKEY = self.key_repr.lowleveltype
-            self.DICTVALUE = self.value_repr.lowleveltype
-
-            # compute the shape of the DICTENTRY structure
-            entryfields = []
-            entrymeths = {
-                'allocate': lltype.typeMethod(_ll_malloc_entries),
-                'delete': _ll_free_entries,
-                'must_clear_key':   (isinstance(self.DICTKEY, lltype.Ptr)
-                                     and self.DICTKEY._needsgc()),
-                'must_clear_value': (isinstance(self.DICTVALUE, lltype.Ptr)
-                                     and self.DICTVALUE._needsgc()),
-                }
-
-            # * the key
-            entryfields.append(("key", self.DICTKEY))
-
-            # * if NULL is not a valid ll value for the key or the value
-            #   field of the entry, it can be used as a marker for
-            #   never-used entries.  Otherwise, we need an explicit flag.
+            DICTKEY = self.key_repr.lowleveltype
+            DICTVALUE = self.value_repr.lowleveltype
+            # * we need an explicit flag if the key and the value is not
+            #   able to store dummy values
             s_key   = self.dictkey.s_value
             s_value = self.dictvalue.s_value
-            nullkeymarker = not self.key_repr.can_ll_be_null(s_key)
-            nullvaluemarker = not self.value_repr.can_ll_be_null(s_value)
-            if self.force_non_null:
-                if not nullkeymarker:
-                    rmodel.warning("%s can be null, but forcing non-null in dict key" % s_key)
-                    nullkeymarker = True
-                if not nullvaluemarker:
-                    rmodel.warning("%s can be null, but forcing non-null in dict value" % s_value)
-                    nullvaluemarker = True
-            dummykeyobj = self.key_repr.get_ll_dummyval_obj(self.rtyper,
-                                                            s_key)
-            dummyvalueobj = self.value_repr.get_ll_dummyval_obj(self.rtyper,
-                                                                s_value)
+            assert not self.force_non_null # XXX kill the flag
+            kwd = {}
+            if self.custom_eq_hash:
+                kwd['get_custom_eq_hash'] = self.custom_eq_hash
+            else:
+                kwd['ll_hash_function'] = self.key_repr.get_ll_hash_function()
+                kwd['ll_eq_function'] = self.key_repr.get_ll_eq_function()
+                kwd['ll_fasthash_function'] = self.key_repr.get_ll_fasthash_function()
+            kwd['dummykeyobj'] = self.key_repr.get_ll_dummyval_obj(self.rtyper,
+                                                                   s_key)
+            kwd['dummyvalueobj'] = self.value_repr.get_ll_dummyval_obj(
+                self.rtyper, s_value)
 
-            # * the state of the entry - trying to encode it as dummy objects
-            if nullkeymarker and dummykeyobj:
-                # all the state can be encoded in the key
-                entrymeths['everused'] = ll_everused_from_key
-                entrymeths['dummy_obj'] = dummykeyobj
-                entrymeths['valid'] = ll_valid_from_key
-                entrymeths['mark_deleted'] = ll_mark_deleted_in_key
-                # the key is overwritten by 'dummy' when the entry is deleted
-                entrymeths['must_clear_key'] = False
-
-            elif nullvaluemarker and dummyvalueobj:
-                # all the state can be encoded in the value
-                entrymeths['everused'] = ll_everused_from_value
-                entrymeths['dummy_obj'] = dummyvalueobj
-                entrymeths['valid'] = ll_valid_from_value
-                entrymeths['mark_deleted'] = ll_mark_deleted_in_value
-                # value is overwritten by 'dummy' when entry is deleted
-                entrymeths['must_clear_value'] = False
-
-            else:
-                # we need a flag to know if the entry was ever used
-                # (we cannot use a NULL as a marker for this, because
-                # the key and value will be reset to NULL to clear their
-                # reference)
-                entryfields.append(("f_everused", lltype.Bool))
-                entrymeths['everused'] = ll_everused_from_flag
-
-                # can we still rely on a dummy obj to mark deleted entries?
-                if dummykeyobj:
-                    entrymeths['dummy_obj'] = dummykeyobj
-                    entrymeths['valid'] = ll_valid_from_key
-                    entrymeths['mark_deleted'] = ll_mark_deleted_in_key
-                    # key is overwritten by 'dummy' when entry is deleted
-                    entrymeths['must_clear_key'] = False
-                elif dummyvalueobj:
-                    entrymeths['dummy_obj'] = dummyvalueobj
-                    entrymeths['valid'] = ll_valid_from_value
-                    entrymeths['mark_deleted'] = ll_mark_deleted_in_value
-                    # value is overwritten by 'dummy' when entry is deleted
-                    entrymeths['must_clear_value'] = False
-                else:
-                    entryfields.append(("f_valid", lltype.Bool))
-                    entrymeths['valid'] = ll_valid_from_flag
-                    entrymeths['mark_deleted'] = ll_mark_deleted_in_flag
-
-            # * the value
-            entryfields.append(("value", self.DICTVALUE))
-
-            # * the hash, if needed
-            if self.custom_eq_hash:
-                fasthashfn = None
-            else:
-                fasthashfn = self.key_repr.get_ll_fasthash_function()
-            if fasthashfn is None:
-                entryfields.append(("f_hash", lltype.Signed))
-                entrymeths['hash'] = ll_hash_from_cache
-            else:
-                entrymeths['hash'] = ll_hash_recomputed
-                entrymeths['fasthashfn'] = fasthashfn
-
-            # Build the lltype data structures
-            self.DICTENTRY = lltype.Struct("dictentry", *entryfields)
-            self.DICTENTRYARRAY = lltype.GcArray(self.DICTENTRY,
-                                                 adtmeths=entrymeths)
-            fields =          [ ("num_items", lltype.Signed),
-                                ("resize_counter", lltype.Signed),
-                                ("entries", lltype.Ptr(self.DICTENTRYARRAY)) ]
-            if self.custom_eq_hash:
-                self.r_rdict_eqfn, self.r_rdict_hashfn = self._custom_eq_hash_repr()
-                fields.extend([ ("fnkeyeq", self.r_rdict_eqfn.lowleveltype),
-                                ("fnkeyhash", self.r_rdict_hashfn.lowleveltype) ])
-                adtmeths = {
-                    'keyhash':        ll_keyhash_custom,
-                    'keyeq':          ll_keyeq_custom,
-                    'r_rdict_eqfn':   self.r_rdict_eqfn,
-                    'r_rdict_hashfn': self.r_rdict_hashfn,
-                    'paranoia':       True,
-                    }
-            else:
-                # figure out which functions must be used to hash and compare
-                ll_keyhash = self.key_repr.get_ll_hash_function()
-                ll_keyeq = self.key_repr.get_ll_eq_function()  # can be None
-                ll_keyhash = lltype.staticAdtMethod(ll_keyhash)
-                if ll_keyeq is not None:
-                    ll_keyeq = lltype.staticAdtMethod(ll_keyeq)
-                adtmeths = {
-                    'keyhash':  ll_keyhash,
-                    'keyeq':    ll_keyeq,
-                    'paranoia': False,
-                    }
-            adtmeths['KEY']   = self.DICTKEY
-            adtmeths['VALUE'] = self.DICTVALUE
-            adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
-            self.DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
-                                             *fields))
-
+            get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT, **kwd)
 
     def convert_const(self, dictobj):
+        XXX
         from rpython.rtyper.lltypesystem import llmemory
         # get object from bound dict methods
         #dictobj = getattr(dictobj, '__self__', dictobj)
@@ -384,36 +383,57 @@
 #  be direct_call'ed from rtyped flow graphs, which means that they will
 #  get flowed and annotated, mostly with SomePtr.
 
-def ll_everused_from_flag(entries, i):
-    return entries[i].f_everused
+DICTINDEX_LONG = lltype.Ptr(lltype.GcArray(lltype.Unsigned))
+DICTINDEX_INT = lltype.Ptr(lltype.GcArray(rffi.UINT))
+DICTINDEX_SHORT = lltype.Ptr(lltype.GcArray(rffi.USHORT))
+DICTINDEX_BYTE = lltype.Ptr(lltype.GcArray(rffi.UCHAR))
 
-def ll_everused_from_key(entries, i):
-    return bool(entries[i].key)
+IS_64BIT = sys.maxint != 2 ** 31 - 1
 
-def ll_everused_from_value(entries, i):
-    return bool(entries[i].value)
+def ll_malloc_indexes_and_choose_lookup(d, n):
+    DICT = lltype.typeOf(d).TO
+    if n <= 256:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_BYTE.TO, n,
+                                                         zero=True))
+        d.lookup_function = DICT.byte_lookup_function
+    elif n <= 65536:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_SHORT.TO, n,
+                                                         zero=True))
+        d.lookup_function = DICT.short_lookup_function
+    elif IS_64BIT and n <= 2 ** 32:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_INT.TO, n,
+                                                         zero=True))
+        d.lookup_function = DICT.int_lookup_function
+    else:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_LONG.TO, n,
+                                                         zero=True))
+        d.lookup_function = DICT.long_lookup_function
 
 def ll_valid_from_flag(entries, i):
     return entries[i].f_valid
 
-def ll_mark_deleted_in_flag(entries, i):
-    entries[i].f_valid = False
-
 def ll_valid_from_key(entries, i):
     ENTRIES = lltype.typeOf(entries).TO
     dummy = ENTRIES.dummy_obj.ll_dummy_value
     return entries.everused(i) and entries[i].key != dummy
 
+def ll_valid_from_value(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    return entries.everused(i) and entries[i].value != dummy
+
+def ll_mark_deleted_in_flag(entries, i):
+    entries[i].f_valid = False
+
 def ll_mark_deleted_in_key(entries, i):
     ENTRIES = lltype.typeOf(entries).TO
     dummy = ENTRIES.dummy_obj.ll_dummy_value
     entries[i].key = dummy
 
-def ll_valid_from_value(entries, i):
-    ENTRIES = lltype.typeOf(entries).TO
-    dummy = ENTRIES.dummy_obj.ll_dummy_value
-    return entries.everused(i) and entries[i].value != dummy
-
 def ll_mark_deleted_in_value(entries, i):
     ENTRIES = lltype.typeOf(entries).TO
     dummy = ENTRIES.dummy_obj.ll_dummy_value
@@ -426,9 +446,6 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
-def ll_get_value(d, i):
-    return d.entries[i].value
-
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
     return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
@@ -445,47 +462,64 @@
     return bool(d) and d.num_items != 0
 
 def ll_dict_getitem(d, key):
-    i = ll_dict_lookup(d, key, d.keyhash(key))
-    if not i & HIGHEST_BIT:
-        return ll_get_value(d, i)
+    index = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
+    if index != -1:
+        return d.entries[index].value
     else:
         raise KeyError
 
 def ll_dict_setitem(d, key, value):
     hash = d.keyhash(key)
-    i = ll_dict_lookup(d, key, hash)
-    return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
+    index = d.lookup_function(d, key, hash, FLAG_STORE)
+    return _ll_dict_setitem_lookup_done(d, key, value, hash, index)
 
 # It may be safe to look inside always, it has a few branches though, and their
 # frequencies needs to be investigated.
 @jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
-    valid = (i & HIGHEST_BIT) == 0
-    i = i & MASK
     ENTRY = lltype.typeOf(d.entries).TO.OF
-    entry = d.entries[i]
-    if not d.entries.everused(i):
-        # a new entry that was never used before
-        ll_assert(not valid, "valid but not everused")
+    if i >= 0:
+        entry = d.entries[i]
+        entry.value = value
+    else:
+        if len(d.entries) == d.num_used_items:
+            ll_dict_grow(d)
+        entry = d.entries[d.num_used_items]
+        entry.key = key
+        entry.value = value
+        if hasattr(ENTRY, 'f_hash'):
+            entry.f_hash = hash
+        if hasattr(ENTRY, 'f_valid'):
+            entry.f_valid = True
+        d.num_used_items += 1
+        d.num_items += 1
         rc = d.resize_counter - 3
-        if rc <= 0:       # if needed, resize the dict -- before the insertion
+        if rc <= 0:
+            XXX
             ll_dict_resize(d)
             i = ll_dict_lookup_clean(d, hash)  # then redo the lookup for 'key'
             entry = d.entries[i]
             rc = d.resize_counter - 3
             ll_assert(rc > 0, "ll_dict_resize failed?")
         d.resize_counter = rc
-        if hasattr(ENTRY, 'f_everused'): entry.f_everused = True
-        entry.value = value
+
+def ll_dict_grow(d):
+    # This over-allocates proportional to the list size, making room
+    # for additional growth.  The over-allocation is mild, but is
+    # enough to give linear-time amortized behavior over a long
+    # sequence of appends() in the presence of a poorly-performing
+    # system malloc().
+    # The growth pattern is:  0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
+    newsize = len(d.entries) + 1
+    if newsize < 9:
+        some = 3
     else:
-        # override an existing or deleted entry
-        entry.value = value
-        if valid:
-            return
-    entry.key = key
-    if hasattr(ENTRY, 'f_hash'):  entry.f_hash = hash
-    if hasattr(ENTRY, 'f_valid'): entry.f_valid = True
-    d.num_items += 1
+        some = 6
+    some += newsize >> 3
+    new_allocated = newsize + some
+    newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+    rgc.ll_arraycopy(d.entries, newitems, 0, 0, len(d.entries))
+    d.entries = newitems
 
 def ll_dict_insertclean(d, key, value, hash):
     # Internal routine used by ll_dict_resize() to insert an item which is
@@ -565,67 +599,103 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
- at jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key))
-def ll_dict_lookup(d, key, hash):
-    entries = d.entries
-    ENTRIES = lltype.typeOf(entries).TO
-    direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
-    mask = len(entries) - 1
-    i = r_uint(hash & mask)
-    # do the first try before any looping
-    if entries.valid(i):
-        checkingkey = entries[i].key
-        if direct_compare and checkingkey == key:
-            return i   # found the entry
-        if d.keyeq is not None and entries.hash(i) == hash:
-            # correct hash, maybe the key is e.g. a different pointer to
-            # an equal object
-            found = d.keyeq(checkingkey, key)
-            if d.paranoia:
-                if (entries != d.entries or
-                    not entries.valid(i) or entries[i].key != checkingkey):
-                    # the compare did major nasty stuff to the dict: start over
-                    return ll_dict_lookup(d, key, hash)
-            if found:
-                return i   # found the entry
-        freeslot = -1
-    elif entries.everused(i):
-        freeslot = intmask(i)
-    else:
-        return i | HIGHEST_BIT # pristine entry -- lookup failed
+FREE = 0
+DELETED = 1
+VALID_OFFSET = 2
 
-    # In the loop, a deleted entry (everused and not valid) is by far
-    # (factor of 100s) the least likely outcome, so test for that last.
-    perturb = r_uint(hash)
-    while 1:
-        # compute the next index using unsigned arithmetic
-        i = (i << 2) + i + perturb + 1
-        i = i & mask
-        # keep 'i' as a signed number here, to consistently pass signed
-        # arguments to the small helper methods.
-        if not entries.everused(i):
-            if freeslot == -1:
-                freeslot = intmask(i)
-            return r_uint(freeslot) | HIGHEST_BIT
-        elif entries.valid(i):
-            checkingkey = entries[i].key
+FLAG_LOOKUP = 0
+FLAG_STORE = 1
+FLAG_DELETE = 2
+
+def new_lookup_function(LOOKUP_FUNC, T):
+    INDEXES = lltype.Ptr(lltype.GcArray(T))
+
+    @jit.look_inside_iff(lambda d, key, hash, store_flag:
+                         jit.isvirtual(d) and jit.isconstant(key))
+    def ll_dict_lookup(d, key, hash, store_flag):
+        entries = d.entries
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        mask = len(indexes) - 1
+        i = hash & mask
+        # do the first try before any looping
+        ENTRIES = lltype.typeOf(entries).TO
+        direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
+        index = rffi.cast(lltype.Signed, indexes[i])
+        if index >= VALID_OFFSET:
+            checkingkey = entries[index - VALID_OFFSET].key
             if direct_compare and checkingkey == key:
-                return i
-            if d.keyeq is not None and entries.hash(i) == hash:
+                XXX
+                return index   # found the entry
+            if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
                 # correct hash, maybe the key is e.g. a different pointer to
                 # an equal object
                 found = d.keyeq(checkingkey, key)
+                #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found)
                 if d.paranoia:
-                    if (entries != d.entries or
-                        not entries.valid(i) or entries[i].key != checkingkey):
-                        # the compare did major nasty stuff to the dict:
-                        # start over
-                        return ll_dict_lookup(d, key, hash)
+                    XXX
+                    if (entries != d.entries or indexes != d.indexes or
+                        not entries.valid(ll_index_getitem(d.size, indexes, i))
+                        or entries.getitem_clean(index).key != checkingkey):
+                        # the compare did major nasty stuff to the dict: start over
+                        if d_signed_indexes(d):
+                            return ll_dict_lookup(d, key, hash,
+                                                  ll_index_getitem_signed)
+                        else:
+                            return ll_dict_lookup(d, key, hash,
+                                                  ll_index_getitem_int)
                 if found:
-                    return i   # found the entry
-        elif freeslot == -1:
-            freeslot = intmask(i)
-        perturb >>= PERTURB_SHIFT
+                    return index - VALID_OFFSET
+            freeslot = -1
+        elif index == DELETED:
+            freeslot = i
+        else:
+            # pristine entry -- lookup failed
+            if store_flag == FLAG_STORE:
+                indexes[i] = rffi.cast(T, d.num_used_items + VALID_OFFSET)
+            return -1
+
+        # In the loop, a deleted entry (everused and not valid) is by far
+        # (factor of 100s) the least likely outcome, so test for that last.
+        XXX
+        perturb = r_uint(hash)
+        while 1:
+            # compute the next index using unsigned arithmetic
+            i = r_uint(i)
+            i = (i << 2) + i + perturb + 1
+            i = intmask(i) & mask
+            index = ll_index_getitem(d.size, indexes, i)
+            # keep 'i' as a signed number here, to consistently pass signed
+            # arguments to the small helper methods.
+            if index == FREE:
+                if freeslot == -1:
+                    freeslot = i
+                return freeslot | HIGHEST_BIT
+            elif entries.valid(index):
+                checkingkey = entries.getitem_clean(index).key
+                if direct_compare and checkingkey == key:
+                    return i
+                if d.keyeq is not None and entries.hash(index) == hash:
+                    # correct hash, maybe the key is e.g. a different pointer to
+                    # an equal object
+                    found = d.keyeq(checkingkey, key)
+                    if d.paranoia:
+                        if (entries != d.entries or indexes != d.indexes or
+                            not entries.valid(ll_index_getitem(d.size, indexes, i)) or
+                            entries.getitem_clean(index).key != checkingkey):
+                            # the compare did major nasty stuff to the dict:
+                            # start over
+                            if d_signed_indexes(d):
+                                return ll_dict_lookup(d, key, hash,
+                                                      ll_index_getitem_signed)
+                            else:
+                                return ll_dict_lookup(d, key, hash,
+                                                      ll_index_getitem_int)
+                    if found:
+                        return i   # found the entry
+            elif freeslot == -1:
+                freeslot = i
+            perturb >>= PERTURB_SHIFT
+    return llhelper(LOOKUP_FUNC, ll_dict_lookup)
 
 def ll_dict_lookup_clean(d, hash):
     # a simplified version of ll_dict_lookup() which assumes that the
@@ -649,12 +719,15 @@
 
 def ll_newdict(DICT):
     d = DICT.allocate()
-    d.entries = DICT.entries.TO.allocate(DICT_INITSIZE)
+    d.entries = DICT.empty_array
+    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
     d.num_items = 0
+    d.num_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
 
 def ll_newdict_size(DICT, length_estimate):
+    xxx
     length_estimate = (length_estimate // 2) * 3
     n = DICT_INITSIZE
     while n < length_estimate:
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -4,7 +4,9 @@
 from rpython.rtyper.lltypesystem import rdict, rstr
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rlib.objectmodel import r_dict
-from rpython.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong
+from rpython.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong,\
+     intmask
+from rpython.rtyper.annlowlevel import llstr, hlstr
 
 import py
 py.log.setconsumer("rtyper", py.log.STDOUT)
@@ -21,6 +23,108 @@
         assert 0 <= x < 4
         yield x
 
+def foreach_index(ll_d):
+    indexes = ll_d.indexes._obj.container._as_ptr()
+    for i in range(len(indexes)):
+        yield rffi.cast(lltype.Signed, indexes[i])
+
+def count_items(ll_d, ITEM):
+    c = 0
+    for item in foreach_index(ll_d):
+        if item == ITEM:
+            c += 1
+    return c
+
+class TestRDictDirect(object):
+    def _get_str_dict(self):
+        # STR -> lltype.Signed
+        DICT = rdict.get_ll_dict(lltype.Ptr(rstr.STR), lltype.Signed,
+                                 ll_fasthash_function=rstr.LLHelpers.ll_strhash,
+                                 ll_hash_function=rstr.LLHelpers.ll_strhash,
+                                 ll_eq_function=rstr.LLHelpers.ll_streq)
+        return DICT
+
+    def test_dict_creation(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("abc"), 13)
+        assert count_items(ll_d, rdict.FREE) == rdict.DICT_INITSIZE - 1
+        assert rdict.ll_dict_getitem(ll_d, llstr("abc")) == 13
+
+    def test_dict_del_lastitem(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        py.test.raises(KeyError, rdict.ll_dict_delitem, ll_d, llstr("abc"))
+        rdict.ll_dict_setitem(ll_d, llstr("abc"), 13)
+        py.test.raises(KeyError, rdict.ll_dict_delitem, ll_d, llstr("def"))
+        rdict.ll_dict_delitem(ll_d, llstr("abc"))
+        assert count_items(ll_d, rdict.FREE) == rdict.DICT_INITSIZE - 1
+        assert count_items(ll_d, rdict.DELETED) == 1
+        py.test.raises(KeyError, rdict.ll_dict_getitem, ll_d, llstr("abc"))
+
+    def test_dict_del_not_lastitem(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("abc"), 13)
+        rdict.ll_dict_setitem(ll_d, llstr("def"), 15)
+        rdict.ll_dict_delitem(ll_d, llstr("abc"))
+        assert count_items(ll_d, rdict.FREE) == rdict.DICT_INITSIZE - 2
+        assert count_items(ll_d, rdict.DELETED) == 1
+
+    def test_dict_resize(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("a"), 1)
+        rdict.ll_dict_setitem(ll_d, llstr("b"), 2)
+        rdict.ll_dict_setitem(ll_d, llstr("c"), 3)
+        rdict.ll_dict_setitem(ll_d, llstr("d"), 4)
+        assert ll_d.size == 8
+        rdict.ll_dict_setitem(ll_d, llstr("e"), 5)
+        rdict.ll_dict_setitem(ll_d, llstr("f"), 6)
+        assert ll_d.size == 32
+        for item in ['a', 'b', 'c', 'd', 'e', 'f']:
+            assert rdict.ll_dict_getitem(ll_d, llstr(item)) == ord(item) - ord('a') + 1
+
+    def test_dict_iteration(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        rdict.ll_dict_setitem(ll_d, llstr("j"), 2)
+        ITER = rdict.get_ll_dictiter(lltype.Ptr(DICT))
+        ll_iter = rdict.ll_dictiter(ITER, ll_d)
+        ll_iterkeys = rdict.ll_dictnext_group['keys']
+        next = ll_iterkeys(lltype.Signed, ll_iter)
+        assert hlstr(next) == "k"
+        next = ll_iterkeys(lltype.Signed, ll_iter)
+        assert hlstr(next) == "j"
+        py.test.raises(StopIteration, ll_iterkeys, lltype.Signed, ll_iter)
+
+    def test_popitem(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        rdict.ll_dict_setitem(ll_d, llstr("j"), 2)
+        ll_elem = rdict.ll_popitem(lltype.Ptr(
+            lltype.GcStruct('x', ('item0', lltype.Ptr(rstr.STR)),
+                            ('item1', lltype.Signed))), ll_d)
+        assert hlstr(ll_elem.item0) == "j"
+        assert ll_elem.item1 == 2
+
+    def test_direct_enter_and_del(self):
+        def eq(a, b):
+            return a == b
+
+        DICT = rdict.get_ll_dict(lltype.Signed, lltype.Signed,
+                                 ll_fasthash_function=intmask,
+                                 ll_hash_function=intmask,
+                                 ll_eq_function=eq)
+        ll_d = rdict.ll_newdict(DICT)
+        numbers = [i * rdict.DICT_INITSIZE + 1 for i in range(8)]
+        for num in numbers:
+            rdict.ll_dict_setitem(ll_d, num, 1)
+            rdict.ll_dict_delitem(ll_d, num)
+            for k in foreach_index(ll_d):
+                assert k < 0
 
 class TestRdict(BaseRtypingTest):
 

From noreply at buildbot.pypy.org  Wed Oct  9 16:01:03 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 16:01:03 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal,
	arigo) cover more cases
Message-ID: <20131009140103.6F8A71C3597@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67245:75f232eda0b4
Date: 2013-10-09 15:58 +0200
http://bitbucket.org/pypy/pypy/changeset/75f232eda0b4/

Log:	(fijal, arigo) cover more cases

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -539,19 +539,19 @@
     d.resize_counter -= 3
 
 def ll_dict_delitem(d, key):
-    i = ll_dict_lookup(d, key, d.keyhash(key))
-    if i & HIGHEST_BIT:
+    index = d.lookup_function(d, key, d.keyhash(key), FLAG_DELETE)
+    if index == -1:
         raise KeyError
-    _ll_dict_del(d, i)
+    _ll_dict_del(d, index)
 
 @jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
-def _ll_dict_del(d, i):
-    d.entries.mark_deleted(i)
+def _ll_dict_del(d, index):
+    d.entries.mark_deleted(index)
     d.num_items -= 1
     # clear the key and the value if they are GC pointers
     ENTRIES = lltype.typeOf(d.entries).TO
     ENTRY = ENTRIES.OF
-    entry = d.entries[i]
+    entry = d.entries[index]
     if ENTRIES.must_clear_key:
         entry.key = lltype.nullptr(ENTRY.key.TO)
     if ENTRIES.must_clear_value:
@@ -624,8 +624,9 @@
         if index >= VALID_OFFSET:
             checkingkey = entries[index - VALID_OFFSET].key
             if direct_compare and checkingkey == key:
-                XXX
-                return index   # found the entry
+                if store_flag == FLAG_DELETE:
+                    indexes[i] = rffi.cast(T, DELETED)
+                return index - VALID_OFFSET   # found the entry
             if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
                 # correct hash, maybe the key is e.g. a different pointer to
                 # an equal object
@@ -644,6 +645,8 @@
                             return ll_dict_lookup(d, key, hash,
                                                   ll_index_getitem_int)
                 if found:
+                    if store_flag == FLAG_DELETE:
+                        indexes[i] = rffi.cast(T, DELETED)
                     return index - VALID_OFFSET
             freeslot = -1
         elif index == DELETED:
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -47,9 +47,15 @@
     def test_dict_creation(self):
         DICT = self._get_str_dict()
         ll_d = rdict.ll_newdict(DICT)
-        rdict.ll_dict_setitem(ll_d, llstr("abc"), 13)
+        lls = llstr("abc")
+        rdict.ll_dict_setitem(ll_d, lls, 13)
         assert count_items(ll_d, rdict.FREE) == rdict.DICT_INITSIZE - 1
         assert rdict.ll_dict_getitem(ll_d, llstr("abc")) == 13
+        assert rdict.ll_dict_getitem(ll_d, lls) == 13
+        rdict.ll_dict_setitem(ll_d, lls, 42)
+        assert rdict.ll_dict_getitem(ll_d, lls) == 42
+        rdict.ll_dict_setitem(ll_d, llstr("abc"), 43)
+        assert rdict.ll_dict_getitem(ll_d, lls) == 43
 
     def test_dict_del_lastitem(self):
         DICT = self._get_str_dict()

From noreply at buildbot.pypy.org  Wed Oct  9 16:11:14 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  9 Oct 2013 16:11:14 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Another TODO item
Message-ID: <20131009141114.0DCBD1C014D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67246:78791c8fb3a3
Date: 2013-10-09 09:07 +0200
http://bitbucket.org/pypy/pypy/changeset/78791c8fb3a3/

Log:	Another TODO item

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -17,3 +17,8 @@
 
 * REDO card marking, starting with "card_page_indices": 128 in
   TRANSLATION_PARAMS
+
+* write barrier: avoid the case when during sweeping we have GCFLAG_VISITED
+  on an object, so we call the slow path, but the slow path doesn't do
+  anything, and we still have GCFLAG_VISITED so we will keep calling it
+  on the same object

From noreply at buildbot.pypy.org  Wed Oct  9 16:11:15 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  9 Oct 2013 16:11:15 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal, arigo)
Message-ID: <20131009141115.6F8631C3597@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67247:8ca34c1e8968
Date: 2013-10-09 16:10 +0200
http://bitbucket.org/pypy/pypy/changeset/8ca34c1e8968/

Log:	(fijal, arigo)

	Copy the logic

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -648,9 +648,9 @@
                     if store_flag == FLAG_DELETE:
                         indexes[i] = rffi.cast(T, DELETED)
                     return index - VALID_OFFSET
-            freeslot = -1
+            deletedslot = -1
         elif index == DELETED:
-            freeslot = i
+            deletedslot = i
         else:
             # pristine entry -- lookup failed
             if store_flag == FLAG_STORE:
@@ -659,29 +659,34 @@
 
         # In the loop, a deleted entry (everused and not valid) is by far
         # (factor of 100s) the least likely outcome, so test for that last.
-        XXX
         perturb = r_uint(hash)
         while 1:
             # compute the next index using unsigned arithmetic
             i = r_uint(i)
             i = (i << 2) + i + perturb + 1
             i = intmask(i) & mask
-            index = ll_index_getitem(d.size, indexes, i)
             # keep 'i' as a signed number here, to consistently pass signed
             # arguments to the small helper methods.
+            index = rffi.cast(lltype.Signed, indexes[i])
             if index == FREE:
-                if freeslot == -1:
-                    freeslot = i
-                return freeslot | HIGHEST_BIT
-            elif entries.valid(index):
-                checkingkey = entries.getitem_clean(index).key
+                if store_flag == FLAG_STORE:
+                    if deletedslot == -1:
+                        deletedslot = i
+                    indexes[deletedslot] = rffi.cast(T, d.num_used_items +
+                                                     VALID_OFFSET)
+                return -1
+            elif index >= VALID_OFFSET:
+                checkingkey = entries[index].key
                 if direct_compare and checkingkey == key:
-                    return i
-                if d.keyeq is not None and entries.hash(index) == hash:
+                    if store_flag == FLAG_DELETE:
+                        indexes[i] = rffi.cast(T, DELETED)
+                    return index - VALID_OFFSET   # found the entry
+                if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
                     # correct hash, maybe the key is e.g. a different pointer to
                     # an equal object
                     found = d.keyeq(checkingkey, key)
                     if d.paranoia:
+                        XXX
                         if (entries != d.entries or indexes != d.indexes or
                             not entries.valid(ll_index_getitem(d.size, indexes, i)) or
                             entries.getitem_clean(index).key != checkingkey):
@@ -694,10 +699,13 @@
                                 return ll_dict_lookup(d, key, hash,
                                                       ll_index_getitem_int)
                     if found:
-                        return i   # found the entry
-            elif freeslot == -1:
-                freeslot = i
+                        if store_flag == FLAG_DELETE:
+                            indexes[i] = rffi.cast(T, DELETED)
+                        return index - VALID_OFFSET
+            elif deletedslot == -1:
+                deletedslot = i
             perturb >>= PERTURB_SHIFT
+
     return llhelper(LOOKUP_FUNC, ll_dict_lookup)
 
 def ll_dict_lookup_clean(d, hash):

From noreply at buildbot.pypy.org  Wed Oct  9 16:53:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  9 Oct 2013 16:53:32 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (arigo,
	fijal around)  in-progress
Message-ID: <20131009145332.B9A891D2321@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67248:0017edf4d84c
Date: 2013-10-09 16:52 +0200
http://bitbucket.org/pypy/pypy/changeset/0017edf4d84c/

Log:	(arigo, fijal around) in-progress

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -99,8 +99,12 @@
     DICTENTRY = lltype.Struct("dictentry", *entryfields)
     DICTENTRYARRAY = lltype.GcArray(DICTENTRY,
                                     adtmeths=entrymeths)
-    LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY, lltype.Signed, lltype.Signed], lltype.Signed))
-
+    LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY,
+                                              lltype.Signed, lltype.Signed],
+                                             lltype.Signed))
+    LOOKCLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
+                                                 lltype.Signed],
+                                                lltype.Signed))
 
     fields =          [ ("num_items", lltype.Signed),
                         ("num_used_items", lltype.Signed),
@@ -135,15 +139,18 @@
     adtmeths['VALUE'] = DICTVALUE
     adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
     adtmeths['empty_array'] = DICTENTRYARRAY.allocate(0)
-    adtmeths['byte_lookup_function'] = new_lookup_function(LOOKUP_FUNC,
-                                                           T=rffi.UCHAR)
-    adtmeths['short_lookup_function'] = new_lookup_function(LOOKUP_FUNC,
-                                                            T=rffi.USHORT)
-    if IS_64BIT:
-        adtmeths['int_lookup_function'] = new_lookup_function(LOOKUP_FUNC,
-                                                              T=rffi.UINT)
-    adtmeths['long_lookup_function'] = new_lookup_function(LOOKUP_FUNC,
-                                                           T=lltype.Unsigned)
+
+    for name, T in [('byte', rffi.UCHAR),
+                    ('short', rffi.USHORT),
+                    ('int', rffi.UINT),
+                    ('long', lltype.Unsigned)]:
+        if name == 'int' and not IS_64BIT:
+            continue
+        lookupfn, lookcleanfn = new_lookup_functions(LOOKUP_FUNC,
+                                                     LOOKCLEAN_FUNC, T=T)
+        adtmeths['%s_lookup_function' % name] = lookupfn
+        adtmeths['%s_lookup_clean_function' % name] = lookcleanfn
+
     DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
                                 *fields))
     return DICT
@@ -397,21 +404,26 @@
                                            lltype.malloc(DICTINDEX_BYTE.TO, n,
                                                          zero=True))
         d.lookup_function = DICT.byte_lookup_function
+        return DICT.byte_lookup_clean_function
     elif n <= 65536:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_SHORT.TO, n,
                                                          zero=True))
         d.lookup_function = DICT.short_lookup_function
+        return DICT.short_lookup_clean_function
     elif IS_64BIT and n <= 2 ** 32:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_INT.TO, n,
                                                          zero=True))
         d.lookup_function = DICT.int_lookup_function
+        return DICT.int_lookup_clean_function
     else:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_LONG.TO, n,
                                                          zero=True))
         d.lookup_function = DICT.long_lookup_function
+        return DICT.long_lookup_clean_function
+ll_malloc_indexes_and_choose_lookup._always_inline_ = True
 
 def ll_valid_from_flag(entries, i):
     return entries[i].f_valid
@@ -495,15 +507,14 @@
         d.num_items += 1
         rc = d.resize_counter - 3
         if rc <= 0:
-            XXX
             ll_dict_resize(d)
-            i = ll_dict_lookup_clean(d, hash)  # then redo the lookup for 'key'
-            entry = d.entries[i]
             rc = d.resize_counter - 3
             ll_assert(rc > 0, "ll_dict_resize failed?")
         d.resize_counter = rc
 
 def ll_dict_grow(d):
+    if d.num_items < d.num_used_items // 4:
+        xxxxxxxxx
     # This over-allocates proportional to the list size, making room
     # for additional growth.  The over-allocation is mild, but is
     # enough to give linear-time amortized behavior over a long
@@ -518,24 +529,37 @@
     some += newsize >> 3
     new_allocated = newsize + some
     newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
-    rgc.ll_arraycopy(d.entries, newitems, 0, 0, len(d.entries))
+    #
+    # XXX we should do this with rgc.ll_arraycopy()!!
+    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
+    i = 0
+    while i < len(d.entries):
+        src = d.entries[i]
+        dst = newitems[i]
+        dst.key = src.key
+        dst.value = src.value
+        if hasattr(ENTRY, 'f_hash'):
+            dst.f_hash = src.f_hash
+        if hasattr(ENTRY, 'f_valid'):
+            dst.f_valid = src.f_valid
+        i += 1
     d.entries = newitems
 
-def ll_dict_insertclean(d, key, value, hash):
+def ll_dict_insertclean(d, key, value, hash, lookcleanfn):
     # Internal routine used by ll_dict_resize() to insert an item which is
     # known to be absent from the dict.  This routine also assumes that
     # the dict contains no deleted entries.  This routine has the advantage
     # of never calling d.keyhash() and d.keyeq(), so it cannot call back
     # to user code.  ll_dict_insertclean() doesn't resize the dict, either.
-    i = ll_dict_lookup_clean(d, hash)
+    index = lookcleanfn(d, hash)
     ENTRY = lltype.typeOf(d.entries).TO.OF
-    entry = d.entries[i]
+    entry = d.entries[index]
     entry.value = value
     entry.key = key
     if hasattr(ENTRY, 'f_hash'):     entry.f_hash = hash
     if hasattr(ENTRY, 'f_valid'):    entry.f_valid = True
-    if hasattr(ENTRY, 'f_everused'): entry.f_everused = True
     d.num_items += 1
+    d.num_used_items += 1
     d.resize_counter -= 3
 
 def ll_dict_delitem(d, key):
@@ -571,29 +595,31 @@
     # avoid extra branches.
 
 def ll_dict_resize(d):
-    old_entries = d.entries
-    old_size = len(old_entries)
     # make a 'new_size' estimate and shrink it if there are many
     # deleted entry markers.  See CPython for why it is a good idea to
     # quadruple the dictionary size as long as it's not too big.
-    num_items = d.num_items + 1
-    if num_items > 50000: new_estimate = num_items * 2
-    else:                 new_estimate = num_items * 4
+    num_items = d.num_used_items
+    if num_items > 50000:
+        new_estimate = num_items * 2
+    else:
+        new_estimate = num_items * 4
     new_size = DICT_INITSIZE
     while new_size <= new_estimate:
         new_size *= 2
+    lookcleanfn = ll_malloc_indexes_and_choose_lookup(d, new_size)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = new_size * 2
     #
-    d.entries = lltype.typeOf(old_entries).TO.allocate(new_size)
-    d.num_items = 0
-    d.resize_counter = new_size * 2
+    entries = d.entries
     i = 0
-    while i < old_size:
-        if old_entries.valid(i):
-            hash = old_entries.hash(i)
-            entry = old_entries[i]
-            ll_dict_insertclean(d, entry.key, entry.value, hash)
+    while i < num_items:
+        if entries.valid(i):
+            hash = entries.hash(i)
+            entry = entries[i]
+            ll_dict_insertclean(d, entry.key, entry.value, hash, lookcleanfn)
         i += 1
-    old_entries.delete()
+    #old_entries.delete() XXXX!
 ll_dict_resize.oopspec = 'dict.resize(d)'
 
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
@@ -607,7 +633,7 @@
 FLAG_STORE = 1
 FLAG_DELETE = 2
 
-def new_lookup_function(LOOKUP_FUNC, T):
+def new_lookup_functions(LOOKUP_FUNC, LOOKCLEAN_FUNC, T):
     INDEXES = lltype.Ptr(lltype.GcArray(T))
 
     @jit.look_inside_iff(lambda d, key, hash, store_flag:
@@ -616,7 +642,7 @@
         entries = d.entries
         indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
         mask = len(indexes) - 1
-        i = hash & mask
+        i = r_uint(hash & mask)
         # do the first try before any looping
         ENTRIES = lltype.typeOf(entries).TO
         direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
@@ -662,7 +688,6 @@
         perturb = r_uint(hash)
         while 1:
             # compute the next index using unsigned arithmetic
-            i = r_uint(i)
             i = (i << 2) + i + perturb + 1
             i = intmask(i) & mask
             # keep 'i' as a signed number here, to consistently pass signed
@@ -706,21 +731,24 @@
                 deletedslot = i
             perturb >>= PERTURB_SHIFT
 
-    return llhelper(LOOKUP_FUNC, ll_dict_lookup)
+    def ll_dict_lookup_clean(d, hash):
+        # a simplified version of ll_dict_lookup() which assumes that the
+        # key is new, and the dictionary doesn't contain deleted entries.
+        # It only finds the next free slot for the given hash.
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        mask = len(indexes) - 1
+        i = r_uint(hash & mask)
+        perturb = r_uint(hash)
+        while rffi.cast(lltype.Signed, indexes[i]) != 0:
+            i = (i << 2) + i + perturb + 1
+            i = i & mask
+            perturb >>= PERTURB_SHIFT
+        index = d.num_used_items
+        indexes[i] = rffi.cast(T, index + VALID_OFFSET)
+        return index
 
-def ll_dict_lookup_clean(d, hash):
-    # a simplified version of ll_dict_lookup() which assumes that the
-    # key is new, and the dictionary doesn't contain deleted entries.
-    # It only finds the next free slot for the given hash.
-    entries = d.entries
-    mask = len(entries) - 1
-    i = r_uint(hash & mask)
-    perturb = r_uint(hash)
-    while entries.everused(i):
-        i = (i << 2) + i + perturb + 1
-        i = i & mask
-        perturb >>= PERTURB_SHIFT
-    return i
+    return (llhelper(LOOKUP_FUNC, ll_dict_lookup),
+            llhelper(LOOKCLEAN_FUNC, ll_dict_lookup_clean))
 
 # ____________________________________________________________
 #
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -23,8 +23,11 @@
         assert 0 <= x < 4
         yield x
 
+def get_indexes(ll_d):
+    return ll_d.indexes._obj.container._as_ptr()
+
 def foreach_index(ll_d):
-    indexes = ll_d.indexes._obj.container._as_ptr()
+    indexes = get_indexes(ll_d)
     for i in range(len(indexes)):
         yield rffi.cast(lltype.Signed, indexes[i])
 
@@ -84,10 +87,10 @@
         rdict.ll_dict_setitem(ll_d, llstr("b"), 2)
         rdict.ll_dict_setitem(ll_d, llstr("c"), 3)
         rdict.ll_dict_setitem(ll_d, llstr("d"), 4)
-        assert ll_d.size == 8
+        assert len(get_indexes(ll_d)) == 8
         rdict.ll_dict_setitem(ll_d, llstr("e"), 5)
         rdict.ll_dict_setitem(ll_d, llstr("f"), 6)
-        assert ll_d.size == 32
+        assert len(get_indexes(ll_d)) == 32
         for item in ['a', 'b', 'c', 'd', 'e', 'f']:
             assert rdict.ll_dict_getitem(ll_d, llstr(item)) == ord(item) - ord('a') + 1
 

From noreply at buildbot.pypy.org  Wed Oct  9 18:24:24 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  9 Oct 2013 18:24:24 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: hg merge default
Message-ID: <20131009162424.691D51C33FE@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67249:8b7f69b48833
Date: 2013-10-09 17:12 +0200
http://bitbucket.org/pypy/pypy/changeset/8b7f69b48833/

Log:	hg merge default

diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -1,1 +1,75 @@
 from _numpypy.numerictypes import *
+import numpypy
+
+def issubclass_(arg1, arg2):
+    """
+    Determine if a class is a subclass of a second class.
+
+    `issubclass_` is equivalent to the Python built-in ``issubclass``,
+    except that it returns False instead of raising a TypeError is one
+    of the arguments is not a class.
+
+    Parameters
+    ----------
+    arg1 : class
+        Input class. True is returned if `arg1` is a subclass of `arg2`.
+    arg2 : class or tuple of classes.
+        Input class. If a tuple of classes, True is returned if `arg1` is a
+        subclass of any of the tuple elements.
+
+    Returns
+    -------
+    out : bool
+        Whether `arg1` is a subclass of `arg2` or not.
+
+    See Also
+    --------
+    issubsctype, issubdtype, issctype
+
+    Examples
+    --------
+    >>> np.issubclass_(np.int32, np.int)
+    True
+    >>> np.issubclass_(np.int32, np.float)
+    False
+
+    """
+    try:
+        return issubclass(arg1, arg2)
+    except TypeError:
+        return False
+
+def issubdtype(arg1, arg2):
+    """
+    Returns True if first argument is a typecode lower/equal in type hierarchy.
+
+    Parameters
+    ----------
+    arg1, arg2 : dtype_like
+        dtype or string representing a typecode.
+
+    Returns
+    -------
+    out : bool
+
+    See Also
+    --------
+    issubsctype, issubclass_
+    numpy.core.numerictypes : Overview of numpy type hierarchy.
+
+    Examples
+    --------
+    >>> np.issubdtype('S1', str)
+    True
+    >>> np.issubdtype(np.float64, np.float32)
+    False
+
+    """
+    if issubclass_(arg2, generic):
+        return issubclass(numpypy.dtype(arg1).type, arg2)
+    mro = numpypy.dtype(arg2).type.mro()
+    if len(mro) > 1:
+        val = mro[1]
+    else:
+        val = mro[0]
+    return issubclass(numpypy.dtype(arg1).type, val)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -400,6 +400,8 @@
     def test_socket_connect_ex(self):
         import _socket
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+        # The following might fail if the DNS redirects failed requests to a
+        # catch-all address (i.e. opendns).
         # Make sure we get an app-level error, not an interp one.
         raises(_socket.gaierror, s.connect_ex, ("wrong.invalid", 80))
         s.close()
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -103,11 +103,13 @@
     exc_p[0] = make_ref(space, operr.w_type)
     val_p[0] = make_ref(space, operr.get_w_value(space))
 
- at cpython_api([], lltype.Void)
+ at cpython_api([], rffi.INT_real, error=0)
 def PyErr_BadArgument(space):
     """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where
     message indicates that a built-in operation was invoked with an illegal
-    argument.  It is mostly for internal use."""
+    argument.  It is mostly for internal use. In CPython this function always
+    raises an exception and returns 0 in all cases, hence the (ab)use of the
+    error indicator."""
     raise OperationError(space.w_TypeError,
             space.wrap("bad argument type for built-in operation"))
 
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -70,9 +70,10 @@
         api.PyErr_Clear()
 
     def test_BadArgument(self, space, api):
-        api.PyErr_BadArgument()
+        ret = api.PyErr_BadArgument()
         state = space.fromcache(State)
         assert space.eq_w(state.operror.w_type, space.w_TypeError)
+        assert ret == 0
         api.PyErr_Clear()
 
     def test_Warning(self, space, api, capfd):
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -91,6 +91,7 @@
         invalid = rffi.str2charp('invalid')
         utf_8 = rffi.str2charp('utf-8')
         prev_encoding = rffi.str2charp(space.unwrap(w_default_encoding))
+        self.raises(space, api, TypeError, api.PyUnicode_SetDefaultEncoding, lltype.nullptr(rffi.CCHARP.TO))
         assert api.PyUnicode_SetDefaultEncoding(invalid) == -1
         assert api.PyErr_Occurred() is space.w_LookupError
         api.PyErr_Clear()
@@ -316,6 +317,15 @@
         rffi.free_charp(b_text)
         rffi.free_charp(b_encoding)
 
+    def test_decode_null_encoding(self, space, api):
+        null_charp = lltype.nullptr(rffi.CCHARP.TO)
+        u_text = u'abcdefg'
+        s_text = space.str_w(api.PyUnicode_AsEncodedString(space.wrap(u_text), null_charp, null_charp))
+        b_text = rffi.str2charp(s_text)
+        assert space.unwrap(api.PyUnicode_Decode(b_text, len(s_text), null_charp, null_charp)) == u_text
+        self.raises(space, api, TypeError, api.PyUnicode_FromEncodedObject, space.wrap(u_text), null_charp, None)
+        rffi.free_charp(b_text)
+
     def test_leak(self):
         size = 50
         raw_buf, gc_buf = rffi.alloc_buffer(size)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -273,6 +273,8 @@
 def PyUnicode_SetDefaultEncoding(space, encoding):
     """Sets the currently active default encoding. Returns 0 on
     success, -1 in case of an error."""
+    if not encoding:
+        PyErr_BadArgument(space)
     w_encoding = space.wrap(rffi.charp2str(encoding))
     setdefaultencoding(space, w_encoding)
     default_encoding[0] = '\x00'
@@ -350,8 +352,11 @@
     in the unicode() built-in function.  The codec to be used is looked up
     using the Python codec registry.  Return NULL if an exception was raised by
     the codec."""
+    if not encoding:
+        # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead
+        encoding = PyUnicode_GetDefaultEncoding(space)
+    w_encoding = space.wrap(rffi.charp2str(encoding))
     w_str = space.wrap(rffi.charpsize2str(s, size))
-    w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))
     else:
@@ -379,6 +384,9 @@
 
     All other objects, including Unicode objects, cause a TypeError to be
     set."""
+    if not encoding:
+        raise OperationError(space.w_TypeError,
+                             space.wrap("decoding Unicode is not supported"))
     w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -319,6 +319,15 @@
         else:
             self.done_func = None
 
+    def are_common_types(self, dtype1, dtype2):
+        if dtype1.is_complex_type() and dtype2.is_complex_type():
+            return True
+        elif not (dtype1.is_complex_type() or dtype2.is_complex_type()) and \
+                (dtype1.is_int_type() and dtype2.is_int_type() or dtype1.is_float_type() and dtype2.is_float_type()) and \
+                not (dtype1.is_bool_type() or dtype2.is_bool_type()):
+            return True
+        return False
+
     @jit.unroll_safe
     def call(self, space, args_w):
         if len(args_w) > 2:
@@ -339,6 +348,12 @@
                  'unsupported operand dtypes %s and %s for "%s"' % \
                  (w_rdtype.get_name(), w_ldtype.get_name(),
                   self.name)))
+
+        if self.are_common_types(w_ldtype, w_rdtype):
+            if not w_lhs.is_scalar() and w_rhs.is_scalar():
+                w_rdtype = w_ldtype
+            elif w_lhs.is_scalar() and not w_rhs.is_scalar():
+                w_ldtype = w_rdtype
         calc_dtype = find_binop_result_dtype(space,
             w_ldtype, w_rdtype,
             int_only=self.int_only,
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2971,6 +2971,11 @@
                         dtype=[('bg', 'i8'), ('fg', 'i8'), ('char', 'S1')])
         assert c[0][0]["char"] == 'a'
 
+    def test_scalar_coercion(self):
+        import numpypy as np
+        a = np.array([1,2,3], dtype=np.int16)
+        assert (a * 2).dtype == np.int16
+
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -149,6 +149,8 @@
         interpleveldefs['nice'] = 'interp_posix.nice'
     if hasattr(os, 'getlogin'):
         interpleveldefs['getlogin'] = 'interp_posix.getlogin'
+    if hasattr(os, 'ctermid'):
+        interpleveldefs['ctermid'] = 'interp_posix.ctermid'
 
     for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid',
                  'seteuid', 'setgid', 'setegid', 'getgroups', 'getpgrp',
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1218,3 +1218,10 @@
         return space.wrap(rurandom.urandom(context, n))
     except OSError, e:
         raise wrap_oserror(space, e)
+
+def ctermid(space):
+    """ctermid() -> string
+
+    Return the name of the controlling terminal for this process.
+    """
+    return space.wrap(os.ctermid())
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -110,7 +110,7 @@
                        '__pypy__', 'cStringIO', '_collections', 'struct',
                        'mmap', 'marshal', '_codecs', 'rctime', 'cppyy',
                        '_cffi_backend', 'pyexpat', '_continuation', '_io',
-                       'thread']:
+                       'thread', 'select']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -49,12 +49,15 @@
     from pypy.module.thread.os_lock import Lock
     assert pypypolicy.look_inside_function(Lock.descr_lock_acquire.im_func)
 
+def test_select():
+    from pypy.module.select.interp_select import poll
+    assert pypypolicy.look_inside_function(poll)
+
 def test_pypy_module():
     from pypy.module._collections.interp_deque import W_Deque
     from pypy.module._random.interp_random import W_Random
     assert not pypypolicy.look_inside_function(W_Random.random)
     assert pypypolicy.look_inside_function(W_Deque.length)
-    assert not pypypolicy.look_inside_pypy_module('select.interp_epoll')
     assert pypypolicy.look_inside_pypy_module('__builtin__.operation')
     assert pypypolicy.look_inside_pypy_module('__builtin__.abstractinst')
     assert pypypolicy.look_inside_pypy_module('__builtin__.functional')
diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -180,12 +180,12 @@
                         i = 0
                         for w_ev in space.listview(w_changelist):
                             ev = space.interp_w(W_Kevent, w_ev)
-                            changelist[i].c_ident = ev.event.c_ident
-                            changelist[i].c_filter = ev.event.c_filter
-                            changelist[i].c_flags = ev.event.c_flags
-                            changelist[i].c_fflags = ev.event.c_fflags
-                            changelist[i].c_data = ev.event.c_data
-                            changelist[i].c_udata = ev.event.c_udata
+                            changelist[i].c_ident = ev.ident
+                            changelist[i].c_filter = ev.filter
+                            changelist[i].c_flags = ev.flags
+                            changelist[i].c_fflags = ev.fflags
+                            changelist[i].c_data = ev.data
+                            changelist[i].c_udata = ev.udata
                             i += 1
                         pchangelist = changelist
                     else:
@@ -206,13 +206,12 @@
                             evt = eventlist[i]
 
                             w_event = W_Kevent(space)
-                            w_event.event = lltype.malloc(kevent, flavor="raw")
-                            w_event.event.c_ident = evt.c_ident
-                            w_event.event.c_filter = evt.c_filter
-                            w_event.event.c_flags = evt.c_flags
-                            w_event.event.c_fflags = evt.c_fflags
-                            w_event.event.c_data = evt.c_data
-                            w_event.event.c_udata = evt.c_udata
+                            w_event.ident = evt.c_ident
+                            w_event.filter = evt.c_filter
+                            w_event.flags = evt.c_flags
+                            w_event.fflags = evt.c_fflags
+                            w_event.data = evt.c_data
+                            w_event.udata = evt.c_udata
 
                             elist_w[i] = w_event
 
@@ -234,11 +233,12 @@
 
 class W_Kevent(W_Root):
     def __init__(self, space):
-        self.event = lltype.nullptr(kevent)
-
-    def __del__(self):
-        if self.event:
-            lltype.free(self.event, flavor="raw")
+        self.ident = rffi.cast(kevent.c_ident, 0)
+        self.filter = rffi.cast(kevent.c_filter, 0)
+        self.flags = rffi.cast(kevent.c_flags, 0)
+        self.fflags = rffi.cast(kevent.c_fflags, 0)
+        self.data = rffi.cast(kevent.c_data, 0)
+        self.udata = lltype.nullptr(rffi.VOIDP.TO)
 
     @unwrap_spec(filter=int, flags='c_uint', fflags='c_uint', data=int, udata=r_uint)
     def descr__init__(self, space, w_ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=r_uint(0)):
@@ -247,35 +247,34 @@
         else:
             ident = r_uint(space.c_filedescriptor_w(w_ident))
 
-        self.event = lltype.malloc(kevent, flavor="raw")
-        rffi.setintfield(self.event, "c_ident", ident)
-        rffi.setintfield(self.event, "c_filter", filter)
-        rffi.setintfield(self.event, "c_flags", flags)
-        rffi.setintfield(self.event, "c_fflags", fflags)
-        rffi.setintfield(self.event, "c_data", data)
-        self.event.c_udata = rffi.cast(rffi.VOIDP, udata)
+        self.ident = rffi.cast(kevent.c_ident, ident)
+        self.filter = rffi.cast(kevent.c_filter, filter)
+        self.flags = rffi.cast(kevent.c_flags, flags)
+        self.fflags = rffi.cast(kevent.c_fflags, fflags)
+        self.data = rffi.cast(kevent.c_data, data)
+        self.udata = rffi.cast(rffi.VOIDP, udata)
 
     def _compare_all_fields(self, other, op):
         if IDENT_UINT:
-            l_ident = rffi.cast(lltype.Unsigned, self.event.c_ident)
-            r_ident = rffi.cast(lltype.Unsigned, other.event.c_ident)
+            l_ident = rffi.cast(lltype.Unsigned, self.ident)
+            r_ident = rffi.cast(lltype.Unsigned, other.ident)
         else:
-            l_ident = self.event.c_ident
-            r_ident = other.event.c_ident
-        l_filter = rffi.cast(lltype.Signed, self.event.c_filter)
-        r_filter = rffi.cast(lltype.Signed, other.event.c_filter)
-        l_flags = rffi.cast(lltype.Unsigned, self.event.c_flags)
-        r_flags = rffi.cast(lltype.Unsigned, other.event.c_flags)
-        l_fflags = rffi.cast(lltype.Unsigned, self.event.c_fflags)
-        r_fflags = rffi.cast(lltype.Unsigned, other.event.c_fflags)
+            l_ident = self.ident
+            r_ident = other.ident
+        l_filter = rffi.cast(lltype.Signed, self.filter)
+        r_filter = rffi.cast(lltype.Signed, other.filter)
+        l_flags = rffi.cast(lltype.Unsigned, self.flags)
+        r_flags = rffi.cast(lltype.Unsigned, other.flags)
+        l_fflags = rffi.cast(lltype.Unsigned, self.fflags)
+        r_fflags = rffi.cast(lltype.Unsigned, other.fflags)
         if IDENT_UINT:
-            l_data = rffi.cast(lltype.Signed, self.event.c_data)
-            r_data = rffi.cast(lltype.Signed, other.event.c_data)
+            l_data = rffi.cast(lltype.Signed, self.data)
+            r_data = rffi.cast(lltype.Signed, other.data)
         else:
-            l_data = self.event.c_data
-            r_data = other.event.c_data
-        l_udata = rffi.cast(lltype.Unsigned, self.event.c_udata)
-        r_udata = rffi.cast(lltype.Unsigned, other.event.c_udata)
+            l_data = self.data
+            r_data = other.data
+        l_udata = rffi.cast(lltype.Unsigned, self.udata)
+        r_udata = rffi.cast(lltype.Unsigned, other.udata)
 
         if op == "eq":
             return l_ident == r_ident and \
@@ -330,22 +329,22 @@
         return space.wrap(self.compare_all_fields(space, w_other, "gt"))
 
     def descr_get_ident(self, space):
-        return space.wrap(self.event.c_ident)
+        return space.wrap(self.ident)
 
     def descr_get_filter(self, space):
-        return space.wrap(self.event.c_filter)
+        return space.wrap(self.filter)
 
     def descr_get_flags(self, space):
-        return space.wrap(self.event.c_flags)
+        return space.wrap(self.flags)
 
     def descr_get_fflags(self, space):
-        return space.wrap(self.event.c_fflags)
+        return space.wrap(self.fflags)
 
     def descr_get_data(self, space):
-        return space.wrap(self.event.c_data)
+        return space.wrap(self.data)
 
     def descr_get_udata(self, space):
-        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.event.c_udata))
+        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.udata))
 
 
 W_Kevent.typedef = TypeDef("select.kevent",
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -30,7 +30,7 @@
 from rpython.rlib import debug, jit, rerased
 from rpython.rlib.listsort import make_timsort_class
 from rpython.rlib.objectmodel import (
-    instantiate, newlist_hint, resizelist_hint, specialize)
+    instantiate, newlist_hint, resizelist_hint, specialize, import_from_mixin)
 from rpython.tool.sourcetools import func_with_new_name
 
 __all__ = ['W_ListObject', 'make_range_list', 'make_empty_list_with_size']
@@ -1170,7 +1170,6 @@
 
 
 class AbstractUnwrappedStrategy(object):
-    _mixin_ = True
 
     def wrap(self, unwrapped):
         raise NotImplementedError
@@ -1329,7 +1328,6 @@
 
     def setslice(self, w_list, start, step, slicelength, w_other):
         assert slicelength >= 0
-        items = self.unerase(w_list.lstorage)
 
         if self is self.space.fromcache(ObjectListStrategy):
             w_other = w_other._temporarily_as_objects()
@@ -1341,6 +1339,7 @@
             w_list.setslice(start, step, slicelength, w_other_as_object)
             return
 
+        items = self.unerase(w_list.lstorage)
         oldsize = len(items)
         len2 = w_other.length()
         if step == 1:  # Support list resizing for non-extended slices
@@ -1456,7 +1455,9 @@
         self.unerase(w_list.lstorage).reverse()
 
 
-class ObjectListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class ObjectListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "object"
 
@@ -1489,7 +1490,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class IntegerListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class IntegerListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0
     _applevel_repr = "int"
 
@@ -1520,7 +1523,30 @@
         return self.unerase(w_list.lstorage)
 
 
-class FloatListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            l = self.unerase(w_list.lstorage)
+            other = w_other.getitems_int()
+            assert other is not None
+            l += other
+            return
+        return self._base_extend_from_list(w_list, w_other)
+
+
+    _base_setslice = setslice
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            storage = self.erase(w_other.getitems_int())
+            w_other = W_ListObject.from_storage_and_strategy(
+                    self.space, storage, self)
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
+class FloatListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0.0
     _applevel_repr = "float"
 
@@ -1548,7 +1574,9 @@
             l.reverse()
 
 
-class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class StringListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "str"
 
@@ -1579,7 +1607,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class UnicodeListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class UnicodeListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "unicode"
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -225,6 +225,15 @@
         l.setslice(0, 1, 2, W_ListObject(space, [w('a'), w(2), w(3)]))
         assert isinstance(l.strategy, ObjectListStrategy)
 
+    def test_setslice_int_range(self):
+        space = self.space
+        w = space.wrap
+        l = W_ListObject(space, [w(1), w(2), w(3)])
+        assert isinstance(l.strategy, IntegerListStrategy)
+        l.setslice(0, 1, 2, make_range_list(space, 5, 1, 4))
+        assert isinstance(l.strategy, IntegerListStrategy)
+
+
     def test_setslice_List(self):
         space = self.space
 
@@ -467,6 +476,12 @@
         l4 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3), self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         assert self.space.eq_w(l3, l4)
 
+    def test_add_of_range_and_int(self):
+        l1 = make_range_list(self.space, 0, 1, 100)
+        l2 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
+        l3 = self.space.add(l2, l1)
+        assert l3.strategy is l2.strategy
+
     def test_mul(self):
         l1 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         l2 = l1.mul(2)
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -10,7 +10,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeFloat, SomeIterator,
     SomePBC, SomeTypedAddressAccess, SomeAddress, SomeType, s_ImpossibleValue,
     s_Bool, s_None, unionof, missing_operation, add_knowntypedata,
-    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString)
+    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper
 from rpython.annotator import builtin
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
@@ -333,12 +333,13 @@
         check_negative_slice(s_start, s_stop)
         lst.listdef.resize()
 
-def check_negative_slice(s_start, s_stop):
+def check_negative_slice(s_start, s_stop, error="slicing"):
     if isinstance(s_start, SomeInteger) and not s_start.nonneg:
-        raise AnnotatorError("slicing: not proven to have non-negative start")
+        raise AnnotatorError("%s: not proven to have non-negative start" %
+                             error)
     if isinstance(s_stop, SomeInteger) and not s_stop.nonneg and \
            getattr(s_stop, 'const', 0) != -1:
-        raise AnnotatorError("slicing: not proven to have non-negative stop")
+        raise AnnotatorError("%s: not proven to have non-negative stop" % error)
 
 
 class __extend__(SomeDict):
@@ -448,12 +449,15 @@
         return s_Bool
 
     def method_find(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "find")
         return SomeInteger()
 
     def method_rfind(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "rfind")
         return SomeInteger()
 
     def method_count(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "count")
         return SomeInteger(nonneg=True)
 
     def method_strip(str, chr):
@@ -520,6 +524,11 @@
     op_contains.can_only_throw = []
 
 
+class __extend__(SomeByteArray):
+    def getslice(ba, s_start, s_stop):
+        check_negative_slice(s_start, s_stop)
+        return SomeByteArray()
+
 class __extend__(SomeUnicodeString):
     def method_encode(uni, s_enc):
         if not s_enc.is_constant():
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -252,6 +252,23 @@
         from rpython.translator.tool.graphpage import try_show
         try_show(self)
 
+    def get_graph(self):
+        import gc
+        pending = [self]   # pending blocks
+        seen = {self: True, None: True}
+        for x in pending:
+            for y in gc.get_referrers(x):
+                if isinstance(y, FunctionGraph):
+                    return y
+                elif isinstance(y, Link):
+                    block = y.prevblock
+                    if block not in seen:
+                        pending.append(block)
+                        seen[block] = True
+                elif isinstance(y, dict):
+                    pending.append(y)   # go back from the dict to the real obj
+        return pending
+
     view = show
 
 
diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py
--- a/rpython/flowspace/specialcase.py
+++ b/rpython/flowspace/specialcase.py
@@ -1,3 +1,4 @@
+import os
 from rpython.flowspace.model import Constant, const
 
 SPECIAL_CASES = {}
@@ -37,6 +38,18 @@
     return space.frame.do_operation('simple_call', const(isinstance),
             w_instance, w_type)
 
+ at register_flow_sc(open)
+def sc_open(space, *args_w):
+    from rpython.rlib.rfile import create_file
+
+    return space.frame.do_operation("simple_call", const(create_file), *args_w)
+
+ at register_flow_sc(os.tmpfile)
+def sc_os_tmpfile(space):
+    from rpython.rlib.rfile import create_temp_rfile
+
+    return space.frame.do_operation("simple_call", const(create_temp_rfile))
+
 # _________________________________________________________________________
 # a simplified version of the basic printing routines, for RPython programs
 class StdOutBuffer:
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1201,6 +1201,7 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+            assert self.card_page_indices > 0
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -1,55 +1,241 @@
 
-""" This file makes open() and friends RPython
+""" This file makes open() and friends RPython. Note that RFile should not
+be used directly and instead it's magically appearing each time you call
+python builtin open()
 """
 
 import os
-from rpython.annotator.model import SomeObject, SomeString, SomeInteger
-from rpython.rtyper.extregistry import ExtRegistryEntry
-from rpython.rtyper.extfunc import register_external
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.tool import rffi_platform as platform
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib import rposix
+from rpython.rlib.rstring import StringBuilder
 
-class SomeFile(SomeObject):
-    def method_write(self, s_arg):
-        assert isinstance(s_arg, SomeString)
+eci = ExternalCompilationInfo(includes=['stdio.h', 'unistd.h', 'sys/types.h'])
 
-    def method_read(self, s_arg=None):
-        if s_arg is not None:
-            assert isinstance(s_arg, SomeInteger)
-        return SomeString(can_be_None=False)
+def llexternal(*args):
+    return rffi.llexternal(*args, compilation_info=eci)
 
-    def method_close(self):
-        pass
+FILE = lltype.Struct('FILE') # opaque type maybe
 
-    def method_seek(self, s_arg, s_whence=None):
-        assert isinstance(s_arg, SomeInteger)
-        if s_whence is not None:
-            assert isinstance(s_whence, SomeInteger)
+class CConfig(object):
+    _compilation_info_ = eci
 
-    def rtyper_makekey(self):
-        return self.__class__,
+    off_t = platform.SimpleType('off_t')
 
-    def rtyper_makerepr(self, rtyper):
-        from rpython.rtyper.lltypesystem.rfile import FileRepr
+CC = platform.configure(CConfig)
+OFF_T = CC['off_t']
 
-        return FileRepr(rtyper)
+c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
+c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
+c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
+                                     lltype.Ptr(FILE)], rffi.SIZE_T)
+c_read = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
+                                   lltype.Ptr(FILE)], rffi.SIZE_T)
+c_feof = llexternal('feof', [lltype.Ptr(FILE)], rffi.INT)
+c_ferror = llexternal('ferror', [lltype.Ptr(FILE)], rffi.INT)
+c_clearerror = llexternal('clearerr', [lltype.Ptr(FILE)], lltype.Void)
+c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT],
+                          rffi.INT)
+c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
+c_fileno = llexternal('fileno', [lltype.Ptr(FILE)], rffi.INT)
+c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], lltype.Signed)
+c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT)
+c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT)
+c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, lltype.Ptr(FILE)],
+                     rffi.CCHARP)
 
-class FileEntry(ExtRegistryEntry):
-    _about_ = open
+BASE_BUF_SIZE = 4096
+BASE_LINE_SIZE = 100
 
-    def compute_result_annotation(self, s_name, s_mode=None):
-        assert isinstance(s_name, SomeString)
-        if s_mode is not None:
-            assert isinstance(s_mode, SomeString)
-        return SomeFile()
+def create_file(filename, mode="r", buffering=-1):
+    assert buffering == -1
+    assert filename is not None
+    assert mode is not None
+    ll_name = rffi.str2charp(filename)
+    try:
+        ll_mode = rffi.str2charp(mode)
+        try:
+            ll_f = c_open(ll_name, ll_mode)
+            if not ll_f:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+        finally:
+            lltype.free(ll_mode, flavor='raw')
+    finally:
+        lltype.free(ll_name, flavor='raw')
+    return RFile(ll_f)
 
-    def specialize_call(self, hop):
-        return hop.r_result.rtype_constructor(hop)
+def create_temp_rfile():
+    res = c_tmpfile()
+    if not res:
+        errno = rposix.get_errno()
+        raise OSError(errno, os.strerror(errno))
+    return RFile(res)
 
-class OSTempfileEntry(ExtRegistryEntry):
-    _about_ = os.tmpfile
+class RFile(object):
+    def __init__(self, ll_file):
+        self.ll_file = ll_file
 
-    def compute_result_annotation(self):
-        return SomeFile()
+    def write(self, value):
+        assert value is not None
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        assert value is not None
+        ll_value = rffi.get_nonmovingbuffer(value)
+        try:
+            # note that since we got a nonmoving buffer, it is either raw
+            # or already cannot move, so the arithmetics below are fine
+            total_bytes = 0
+            ll_current = ll_value
+            while total_bytes < len(value):
+                bytes = c_write(ll_current, 1, len(value) - r_uint(total_bytes),
+                                ll_file)
+                if bytes == 0:
+                    errno = rposix.get_errno()
+                    raise OSError(errno, os.strerror(errno))
+                total_bytes += bytes
+                ll_current = rffi.cast(rffi.CCHARP,
+                                       rffi.cast(lltype.Unsigned, ll_value) +
+                                       total_bytes)
+        finally:
+            rffi.free_nonmovingbuffer(value, ll_value)
 
-    def specialize_call(self, hop):
-        return hop.r_result.rtype_tempfile(hop)
+    def close(self):
+        if self.ll_file:
+            # double close is allowed
+            res = c_close(self.ll_file)
+            self.ll_file = lltype.nullptr(FILE)
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
 
+    def read(self, size=-1):
+        # XXX CPython uses a more delicate logic here
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        if size < 0:
+            # read the entire contents
+            buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
+            try:
+                s = StringBuilder()
+                while True:
+                    returned_size = c_read(buf, 1, BASE_BUF_SIZE, ll_file)
+                    if returned_size == 0:
+                        if c_feof(ll_file):
+                            # ok, finished
+                            return s.build()
+                        errno = c_ferror(ll_file)
+                        c_clearerror(ll_file)
+                        raise OSError(errno, os.strerror(errno))
+                    s.append_charpsize(buf, returned_size)
+            finally:
+                lltype.free(buf, flavor='raw')
+        else:
+            raw_buf, gc_buf = rffi.alloc_buffer(size)
+            try:
+                returned_size = c_read(raw_buf, 1, size, ll_file)
+                if returned_size == 0:
+                    if not c_feof(ll_file):
+                        errno = c_ferror(ll_file)
+                        raise OSError(errno, os.strerror(errno))
+                s = rffi.str_from_buffer(raw_buf, gc_buf, size,
+                                         rffi.cast(lltype.Signed, returned_size))
+            finally:
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+            return s
+
+    def seek(self, pos, whence=0):
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        res = c_fseek(ll_file, pos, whence)
+        if res == -1:
+            errno = rposix.get_errno()
+            raise OSError(errno, os.strerror(errno))
+
+    def fileno(self):
+        if self.ll_file:
+            return intmask(c_fileno(self.ll_file))
+        raise ValueError("I/O operation on closed file")
+
+    def tell(self):
+        if self.ll_file:
+            res = intmask(c_ftell(self.ll_file))
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return res
+        raise ValueError("I/O operation on closed file")
+
+    def flush(self):
+        if self.ll_file:
+            res = c_fflush(self.ll_file)
+            if res != 0:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return
+        raise ValueError("I/O operation on closed file")
+
+    def truncate(self, arg=-1):
+        if self.ll_file:
+            if arg == -1:
+                arg = self.tell()
+            res = c_ftruncate(self.fileno(), arg)
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return
+        raise ValueError("I/O operation on closed file")
+
+    def __del__(self):
+        self.close()
+
+    def _readline1(self, raw_buf):
+        result = c_fgets(raw_buf, BASE_LINE_SIZE, self.ll_file)
+        if not result:
+            if c_feof(self.ll_file):   # ok
+                return 0
+            errno = c_ferror(self.ll_file)
+            raise OSError(errno, os.strerror(errno))
+        #
+        # Assume that fgets() works as documented, and additionally
+        # never writes beyond the final \0, which the CPython
+        # fileobject.c says appears to be the case everywhere.
+        # The only case where the buffer was not big enough is the
+        # case where the buffer is full, ends with \0, and doesn't
+        # end with \n\0.
+        strlen = 0
+        while raw_buf[strlen] != '\0':
+            strlen += 1
+        if (strlen == BASE_LINE_SIZE - 1 and
+              raw_buf[BASE_LINE_SIZE - 2] != '\n'):
+            return -1    # overflow!
+        # common case
+        return strlen
+
+    def readline(self):
+        if self.ll_file:
+            raw_buf, gc_buf = rffi.alloc_buffer(BASE_LINE_SIZE)
+            try:
+                c = self._readline1(raw_buf)
+                if c >= 0:
+                    return rffi.str_from_buffer(raw_buf, gc_buf,
+                                                BASE_LINE_SIZE, c)
+                #
+                # this is the rare case: the line is longer than BASE_LINE_SIZE
+                s = StringBuilder()
+                while True:
+                    s.append_charpsize(raw_buf, BASE_LINE_SIZE - 1)
+                    c = self._readline1(raw_buf)
+                    if c >= 0:
+                        break
+                #
+                s.append_charpsize(raw_buf, c)
+                return s.build()
+            finally:
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+        raise ValueError("I/O operation on closed file")
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -78,3 +78,100 @@
 
         f()
         self.interpret(f, [])
+
+    def test_fileno(self):
+        fname = str(self.tmpdir.join('file_5'))
+
+        def f():
+            f = open(fname, "w")
+            try:
+                return f.fileno()
+            finally:
+                f.close()
+
+        res = self.interpret(f, [])
+        assert res > 2
+
+    def test_tell(self):
+        fname = str(self.tmpdir.join('file_tell'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            try:
+                return f.tell()
+            finally:
+                f.close()
+
+        res = self.interpret(f, [])
+        assert res == 3
+
+    def test_flush(self):
+        fname = str(self.tmpdir.join('file_flush'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            f.flush()
+            f2 = open(fname)
+            assert f2.read() == "xyz"
+            f2.close()
+            f.close()
+
+        self.interpret(f, [])
+
+    def test_truncate(self):
+        fname = str(self.tmpdir.join('file_trunc'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            f.seek(0)
+            f.truncate(2)
+            f.close()
+            f2 = open(fname)
+            assert f2.read() == "xy"
+            f2.close()
+
+        f()
+        self.interpret(f, [])
+
+
+class TestDirect:
+    def setup_class(cls):
+        cls.tmpdir = udir.join('test_rfile_direct')
+        cls.tmpdir.ensure(dir=True)
+
+    def test_readline(self):
+        fname = str(self.tmpdir.join('file_readline'))
+        j = 0
+        expected = []
+        with open(fname, 'w') as f:
+            for i in range(250):
+                s = ''.join([chr(32+(k&63)) for k in range(j, j + i)])
+                j += 1
+                print >> f, s
+        expected = open(fname).readlines()
+        expected += ['', '']
+        assert len(expected) == 252
+
+        f = rfile.create_file(fname, 'r')
+        for j in range(252):
+            got = f.readline()
+            assert got == expected[j]
+        f.close()
+
+    def test_readline_without_eol_at_the_end(self):
+        fname = str(self.tmpdir.join('file_readline_without_eol_at_the_end'))
+        for n in [1, 10, 97, 98, 99, 100, 101, 102, 103, 150,
+                  196, 197, 198, 199, 200, 201, 202, 203, 204, 250]:
+            s = ''.join([chr(32+(k&63)) for k in range(n)])
+            with open(fname, 'wb') as f:
+                f.write(s)
+
+            f = rfile.create_file(fname, 'r')
+            got = f.readline()
+            assert got == s
+            got = f.readline()
+            assert got == ''
+            f.close()
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -301,6 +301,8 @@
             addr.get_port() == 80):
             found = True
     assert found, lst
+    # The following might fail if the DNS redirects failed requests to a
+    # catch-all address (i.e. opendns).
     e = py.test.raises(GAIError, getaddrinfo, 'www.very-invalidaddress.com', None)
     assert isinstance(e.value.get_msg(), str)
 
diff --git a/rpython/rlib/types.py b/rpython/rlib/types.py
--- a/rpython/rlib/types.py
+++ b/rpython/rlib/types.py
@@ -39,8 +39,12 @@
     return model.SomeUnicodeString(no_nul=True)
 
 
-def str():
-    return model.SomeString()
+def str(can_be_None=False):
+    return model.SomeString(can_be_None=can_be_None)
+
+
+def bytearray():
+    return model.SomeByteArray()
 
 
 def str0():
diff --git a/rpython/rtyper/lltypesystem/rbytearray.py b/rpython/rtyper/lltypesystem/rbytearray.py
--- a/rpython/rtyper/lltypesystem/rbytearray.py
+++ b/rpython/rtyper/lltypesystem/rbytearray.py
@@ -16,6 +16,9 @@
                                                                   lltype.Char,
                                                                   'bytearray_from_str')
 
+def _empty_bytearray():
+    return empty
+
 BYTEARRAY.become(lltype.GcStruct('rpy_bytearray',
                  ('chars', lltype.Array(lltype.Char)), adtmeths={
     'malloc' : lltype.staticAdtMethod(mallocbytearray),
@@ -23,8 +26,11 @@
     'copy_contents_from_str': lltype.staticAdtMethod(
                                          copy_bytearray_contents_from_str),
     'length': rstr.LLHelpers.ll_length,
+    'empty': lltype.staticAdtMethod(_empty_bytearray),
 }))
 
+empty = lltype.malloc(BYTEARRAY, 0, immortal=True)
+
 class LLHelpers(rstr.LLHelpers):
     @classmethod
     def ll_strsetitem(cls, s, i, item):
diff --git a/rpython/rtyper/lltypesystem/rfile.py b/rpython/rtyper/lltypesystem/rfile.py
deleted file mode 100644
--- a/rpython/rtyper/lltypesystem/rfile.py
+++ /dev/null
@@ -1,195 +0,0 @@
-
-import os
-from rpython.rlib import rposix
-from rpython.rlib.rarithmetic import r_uint
-from rpython.annotator import model as annmodel
-from rpython.rtyper.rtyper import Repr
-from rpython.rlib.rstring import StringBuilder
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
-from rpython.rtyper.lltypesystem.rstr import string_repr, STR
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.annlowlevel import hlstr
-from rpython.rtyper.lltypesystem.lloperation import llop
-
-FILE = lltype.Struct('FILE') # opaque type maybe
-FILE_WRAPPER = lltype.GcStruct("FileWrapper", ('file', lltype.Ptr(FILE)))
-
-eci = ExternalCompilationInfo(includes=['stdio.h'])
-
-def llexternal(*args):
-    return rffi.llexternal(*args, compilation_info=eci)
-
-c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
-c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
-c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
-                                     lltype.Ptr(FILE)], rffi.SIZE_T)
-c_read = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
-                                   lltype.Ptr(FILE)], rffi.SIZE_T)
-c_feof = llexternal('feof', [lltype.Ptr(FILE)], rffi.INT)
-c_ferror = llexternal('ferror', [lltype.Ptr(FILE)], rffi.INT)
-c_clearerror = llexternal('clearerr', [lltype.Ptr(FILE)], lltype.Void)
-c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT],
-                          rffi.INT)
-c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
-
-def ll_open(name, mode):
-    file_wrapper = lltype.malloc(FILE_WRAPPER)
-    ll_name = rffi.str2charp(name)
-    ll_mode = rffi.str2charp(mode)
-    try:
-        ll_f = c_open(ll_name, ll_mode)
-        if not ll_f:
-            errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
-        file_wrapper.file = ll_f
-    finally:
-        lltype.free(ll_name, flavor='raw')
-        lltype.free(ll_mode, flavor='raw')
-    return file_wrapper
-
-def ll_tmpfile():
-    file_wrapper = lltype.malloc(FILE_WRAPPER)
-    res = c_tmpfile()
-    if not res:
-        errno = rposix.get_errno()
-        raise OSError(errno, os.strerror(errno))
-    file_wrapper.file = res
-    return file_wrapper
-
-def ll_write(file_wrapper, value):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    value = hlstr(value)
-    assert value is not None
-    ll_value = rffi.get_nonmovingbuffer(value)
-    try:
-        # note that since we got a nonmoving buffer, it is either raw
-        # or already cannot move, so the arithmetics below are fine
-        total_bytes = 0
-        ll_current = ll_value
-        while total_bytes < len(value):
-            bytes = c_write(ll_current, 1, len(value) - r_uint(total_bytes),
-                            ll_file)
-            if bytes == 0:
-                errno = rposix.get_errno()
-                raise OSError(errno, os.strerror(errno))
-            total_bytes += bytes
-            ll_current = rffi.cast(rffi.CCHARP,
-                                   rffi.cast(lltype.Unsigned, ll_value) +
-                                   total_bytes)
-    finally:
-        rffi.free_nonmovingbuffer(value, ll_value)
-
-BASE_BUF_SIZE = 4096
-
-def ll_read(file_wrapper, size):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    if size < 0:
-        # read the entire contents
-        buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
-        try:
-            s = StringBuilder()
-            while True:
-                returned_size = c_read(buf, 1, BASE_BUF_SIZE, ll_file)
-                if returned_size == 0:
-                    if c_feof(ll_file):
-                        # ok, finished
-                        return s.build()
-                    errno = c_ferror(ll_file)
-                    c_clearerror(ll_file)
-                    raise OSError(errno, os.strerror(errno))
-                s.append_charpsize(buf, returned_size)
-        finally:
-            lltype.free(buf, flavor='raw')
-    else:
-        raw_buf, gc_buf = rffi.alloc_buffer(size)
-        try:
-            returned_size = c_read(raw_buf, 1, size, ll_file)
-            if returned_size == 0:
-                if not c_feof(ll_file):
-                    errno = c_ferror(ll_file)
-                    raise OSError(errno, os.strerror(errno))
-            s = rffi.str_from_buffer(raw_buf, gc_buf, size,
-                                     rffi.cast(lltype.Signed, returned_size))
-        finally:
-            rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
-        return s
-def ll_seek(file_wrapper, pos, whence):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    res = c_fseek(ll_file, pos, whence)
-    if res == -1:
-        errno = rposix.get_errno()
-        raise OSError(errno, os.strerror(errno))
-
-def ll_close(file_wrapper):
-    if file_wrapper.file:
-        # double close is allowed
-        res = c_close(file_wrapper.file)
-        file_wrapper.file = lltype.nullptr(FILE)
-        if res == -1:
-            errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
-
-class FileRepr(Repr):
-    lowleveltype = lltype.Ptr(FILE_WRAPPER)
-
-    def __init__(self, typer):
-        Repr.__init__(self)
-
-    def rtype_constructor(self, hop):
-        repr = hop.rtyper.getrepr(annmodel.SomeString())
-        arg_0 = hop.inputarg(repr, 0)
-        if len(hop.args_v) == 1:
-            arg_1 = hop.inputconst(string_repr, "r")
-        else:
-            arg_1 = hop.inputarg(repr, 1)
-        hop.exception_is_here()
-        open = hop.rtyper.getannmixlevel().delayedfunction(
-            ll_open, [annmodel.SomeString()] * 2,
-            annmodel.SomePtr(self.lowleveltype))
-        v_open = hop.inputconst(lltype.typeOf(open), open)
-        return hop.genop('direct_call', [v_open, arg_0, arg_1],
-                         resulttype=self)
-
-    def rtype_tempfile(self, hop):
-        tmpfile = hop.rtyper.getannmixlevel().delayedfunction(
-            ll_tmpfile, [], annmodel.SomePtr(self.lowleveltype))
-        v_tmpfile = hop.inputconst(lltype.typeOf(tmpfile), tmpfile)
-        hop.exception_is_here()
-        return hop.genop('direct_call', [v_tmpfile], resulttype=self)
-
-
-    def rtype_method_write(self, hop):
-        args_v = hop.inputargs(self, string_repr)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_write, *args_v)
-
-    def rtype_method_close(self, hop):
-        r_self = hop.inputarg(self, 0)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_close, r_self)
-
-    def rtype_method_read(self, hop):
-        r_self = hop.inputarg(self, 0)
-        if len(hop.args_v) != 2:
-            arg_1 = hop.inputconst(lltype.Signed, -1)
-        else:
-            arg_1 = hop.inputarg(lltype.Signed, 1)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_read, r_self, arg_1)
-
-    def rtype_method_seek(self, hop):
-        r_self = hop.inputarg(self, 0)
-        arg_1 = hop.inputarg(lltype.Signed, 1)
-        if len(hop.args_v) != 3:
-            arg_2 = hop.inputconst(lltype.Signed, os.SEEK_SET)
-        else:
-            arg_2 = hop.inputarg(lltype.Signed, 2)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_seek, r_self, arg_1, arg_2)
-
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -1685,7 +1685,7 @@
         def tmpnam_llimpl():
             return rffi.charp2str(os_tmpnam(lltype.nullptr(rffi.CCHARP.TO)))
 
-        return extdef([], None, llimpl=tmpnam_llimpl,
+        return extdef([], str, llimpl=tmpnam_llimpl,
                       export_name="ll_os.ll_os_tmpnam")
 
 # --------------------------- os.stat & variants ---------------------------
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -187,13 +187,15 @@
         if hop.nb_args > 2:
             v_start = hop.inputarg(Signed, arg=2)
             if not hop.args_s[2].nonneg:
-                raise TyperError("str.find() start must be proven non-negative")
+                raise TyperError("str.%s() start must be proven non-negative"
+                                 % (reverse and 'rfind' or 'find',))
         else:
             v_start = hop.inputconst(Signed, 0)
         if hop.nb_args > 3:
             v_end = hop.inputarg(Signed, arg=3)
             if not hop.args_s[3].nonneg:
-                raise TyperError("str.find() end must be proven non-negative")
+                raise TyperError("str.%s() end must be proven non-negative"
+                                 % (reverse and 'rfind' or 'find',))
         else:
             v_end = hop.gendirectcall(self.ll.ll_strlen, v_str)
         hop.exception_cannot_occur()
diff --git a/rpython/rtyper/test/test_rbytearray.py b/rpython/rtyper/test/test_rbytearray.py
--- a/rpython/rtyper/test/test_rbytearray.py
+++ b/rpython/rtyper/test/test_rbytearray.py
@@ -50,3 +50,10 @@
 
         ll_res = self.interpret(f, [123])
         assert hlstr(ll_res) == "123"
+
+    def test_getslice(self):
+        def f(x):
+            return str(bytearray(str(x))[1:2])
+
+        ll_res = self.interpret(f, [123])
+        assert hlstr(ll_res) == "2"
diff --git a/rpython/rtyper/test/test_rint.py b/rpython/rtyper/test/test_rint.py
--- a/rpython/rtyper/test/test_rint.py
+++ b/rpython/rtyper/test/test_rint.py
@@ -85,6 +85,14 @@
         res = self.ll_to_string(res)
         assert res == '-0x8' + '0' * (len(res)-4)
 
+    def test_hex_of_uint(self):
+        def dummy(i):
+            return hex(r_uint(i))
+
+        res = self.interpret(dummy, [-5])
+        res = self.ll_to_string(res)
+        assert res == '0x' + 'f' * (len(res)-3) + 'b'
+
     def test_oct_of_int(self):
         def dummy(i):
             return oct(i)
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -416,6 +416,14 @@
             res = self.interpret(f, [i])
             assert res == expected
 
+    def test_rfind_error_message(self):
+        const = self.const
+        def f(i):
+            return const("abc").rfind(const(''), i)
+        e = py.test.raises(TyperError, self.interpret, f, [-5])
+        assert str(e.value).startswith(
+            'str.rfind() start must be proven non-negative')
+
     def test_find_char(self):
         const = self.const
         def fn(ch):
@@ -1134,4 +1142,4 @@
         array = lltype.malloc(TP, 12, flavor='raw')
         self.interpret(f, [array, 4])
         assert list(array) == list('abc'*4)
-        lltype.free(array, flavor='raw')
\ No newline at end of file
+        lltype.free(array, flavor='raw')
diff --git a/rpython/translator/c/src/stacklet/switch_arm_gcc.h b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
--- a/rpython/translator/c/src/stacklet/switch_arm_gcc.h
+++ b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
@@ -1,9 +1,8 @@
-#if __ARM_ARCH__ >= 5
-# define call_reg(x) "blx " #x "\n"
-#elif defined (__ARM_ARCH_4T__)
+#if defined(__ARM_ARCH_4__) || defined (__ARM_ARCH_4T__)
 # define call_reg(x) "mov lr, pc ; bx " #x "\n"
 #else
-# define call_reg(x) "mov lr, pc ; mov pc, " #x "\n"
+/* ARM >= 5 */
+# define call_reg(x) "blx " #x "\n"
 #endif
 
 static void __attribute__((optimize("O3"))) *slp_switch(void *(*save_state)(void*, void*),
diff --git a/rpython/translator/tool/graphpage.py b/rpython/translator/tool/graphpage.py
--- a/rpython/translator/tool/graphpage.py
+++ b/rpython/translator/tool/graphpage.py
@@ -200,7 +200,7 @@
                 dotgen.emit_edge(nameof(cdef), nameof(prevcdef), color="red")
             prevcdef = cdef
             cdef = cdef.basedef
-        
+
         self.source = dotgen.generate(target=None)
 
     def followlink(self, name):
@@ -224,7 +224,7 @@
         dotgen.emit('mclimit=15.0')
 
         self.do_compute(dotgen, *args, **kwds)
-        
+
         self.source = dotgen.generate(target=None)
 
         # link the function names to the individual flow graphs
@@ -264,7 +264,7 @@
                 data = self.labelof(classdef, classdef.shortname)
                 dotgen.emit_node(nameof(classdef), label=data, shape="box")
                 dotgen.emit_edge(nameof(classdef.basedef), nameof(classdef))
-             
+
     def labelof(self, obj, objname):
         name = objname
         i = 1
@@ -409,22 +409,11 @@
     elif isinstance(obj, Link):
         try_show(obj.prevblock)
     elif isinstance(obj, Block):
-        import gc
-        pending = [obj]   # pending blocks
-        seen = {obj: True, None: True}
-        for x in pending:
-            for y in gc.get_referrers(x):
-                if isinstance(y, FunctionGraph):
-                    y.show()
-                    return
-                elif isinstance(y, Link):
-                    block = y.prevblock
-                    if block not in seen:
-                        pending.append(block)
-                        seen[block] = True
-                elif isinstance(y, dict):
-                    pending.append(y)   # go back from the dict to the real obj
-        graph = IncompleteGraph(pending)
+        graph = obj.get_graph()
+        if isinstance(graph, FunctionGraph):
+            graph.show()
+            return
+        graph = IncompleteGraph(graph)
         SingleGraphPage(graph).display()
     else:
         raise TypeError("try_show(%r object)" % (type(obj).__name__,))
@@ -449,7 +438,7 @@
                         seen[block] = True
         return pending
     else:
-        raise TypeError("try_get_functiongraph(%r object)" % (type(obj).__name__,))    
+        raise TypeError("try_get_functiongraph(%r object)" % (type(obj).__name__,))
 
 class IncompleteGraph:
     name = '(incomplete graph)'

From noreply at buildbot.pypy.org  Wed Oct  9 18:24:28 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  9 Oct 2013 18:24:28 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: implement the fast-path for
 intstrategy and long[] only
Message-ID: <20131009162428.9A4A51C369D@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67250:a8d55ebf78ea
Date: 2013-10-09 18:20 +0200
http://bitbucket.org/pypy/pypy/changeset/a8d55ebf78ea/

Log:	implement the fast-path for intstrategy and long[] only

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -85,6 +85,11 @@
             return self.space.wrap(s)
         return W_CType.string(self, cdataobj, maxlen)
 
+    def is_long(self):
+        return False
+
+    def is_double(self):
+        return False
 
 class W_CTypePrimitiveCharOrUniChar(W_CTypePrimitive):
     _attrs_ = []
@@ -171,6 +176,9 @@
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
 
+    def is_long(self):
+        return self.size == rffi.sizeof(lltype.Signed)
+
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
 
@@ -274,6 +282,9 @@
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
 
+    def is_double(self):
+        return self.size == rffi.sizeof(lltype.Float)
+
     def cast(self, w_ob):
         space = self.space
         if isinstance(w_ob, cdataobj.W_CData):
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -58,19 +58,44 @@
             value = rffi.cast(rffi.CCHARP, value)
         return cdataobj.W_CData(space, value, self)
 
+    def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
+        from rpython.rlib.rarray import copy_list_to_raw_array
+        from pypy.objspace.std.listobject import W_ListObject, IntegerListStrategy
+        if not isinstance(w_ob, W_ListObject):
+            return False
+        #
+        int_stragegy = self.space.fromcache(IntegerListStrategy)
+
+        if w_ob.strategy is int_stragegy and self.ctitem.is_long():
+            int_list = w_ob.strategy.unerase(w_ob.lstorage)
+            cdata = rffi.cast(rffi.LONGP, cdata)
+            copy_list_to_raw_array(int_list, cdata)
+            return True
+
+        return False
+
+    def _convert_array_from_listview(self, cdata, w_ob):
+        space = self.space
+        lst_w = space.listview(w_ob)
+        if self.length >= 0 and len(lst_w) > self.length:
+            raise operationerrfmt(space.w_IndexError,
+                "too many initializers for '%s' (got %d)",
+                                  self.name, len(lst_w))
+        ctitem = self.ctitem
+        for i in range(len(lst_w)):
+            ctitem.convert_from_object(cdata, lst_w[i])
+            cdata = rffi.ptradd(cdata, ctitem.size)
+
     def convert_array_from_object(self, cdata, w_ob):
         space = self.space
+        if self._convert_array_from_list_strategy_maybe(cdata, w_ob):
+            # the fast path worked, we are done now
+            return
+        #
+        # continue with the slow path
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
-            lst_w = space.listview(w_ob)
-            if self.length >= 0 and len(lst_w) > self.length:
-                raise operationerrfmt(space.w_IndexError,
-                    "too many initializers for '%s' (got %d)",
-                                      self.name, len(lst_w))
-            ctitem = self.ctitem
-            for i in range(len(lst_w)):
-                ctitem.convert_from_object(cdata, lst_w[i])
-                cdata = rffi.ptradd(cdata, ctitem.size)
+            self._convert_array_from_listview(cdata, w_ob)
         elif (self.can_cast_anything or
               (self.ctitem.is_primitive_integer and
                self.ctitem.size == rffi.sizeof(lltype.Char))):
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -139,6 +139,8 @@
 
 class W_ListObject(W_Root):
 
+    strategy = None
+
     def __init__(self, space, wrappeditems, sizehint=-1):
         assert isinstance(wrappeditems, list)
         self.space = space

From noreply at buildbot.pypy.org  Wed Oct  9 18:24:32 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  9 Oct 2013 18:24:32 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: implement the fast-path
 also for double[] and floatstrategy
Message-ID: <20131009162432.D203A1C36AC@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67251:e3f5920e0686
Date: 2013-10-09 18:23 +0200
http://bitbucket.org/pypy/pypy/changeset/e3f5920e0686/

Log:	implement the fast-path also for double[] and floatstrategy

diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -60,18 +60,26 @@
 
     def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
         from rpython.rlib.rarray import copy_list_to_raw_array
-        from pypy.objspace.std.listobject import W_ListObject, IntegerListStrategy
+        from pypy.objspace.std.listobject import (W_ListObject,
+                                                  IntegerListStrategy, FloatListStrategy)
         if not isinstance(w_ob, W_ListObject):
             return False
         #
         int_stragegy = self.space.fromcache(IntegerListStrategy)
-
+        float_strategy = self.space.fromcache(FloatListStrategy)
+        #
         if w_ob.strategy is int_stragegy and self.ctitem.is_long():
             int_list = w_ob.strategy.unerase(w_ob.lstorage)
             cdata = rffi.cast(rffi.LONGP, cdata)
             copy_list_to_raw_array(int_list, cdata)
             return True
-
+        #
+        if w_ob.strategy is float_strategy and self.ctitem.is_double():
+            float_list = w_ob.strategy.unerase(w_ob.lstorage)
+            cdata = rffi.cast(rffi.DOUBLEP, cdata)
+            copy_list_to_raw_array(float_list, cdata)
+            return True
+        #
         return False
 
     def _convert_array_from_listview(self, cdata, w_ob):

From noreply at buildbot.pypy.org  Wed Oct  9 18:37:39 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  9 Oct 2013 18:37:39 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal, arigo)
Message-ID: <20131009163739.1FF591C369D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67252:5d7877739b59
Date: 2013-10-09 18:36 +0200
http://bitbucket.org/pypy/pypy/changeset/5d7877739b59/

Log:	(fijal, arigo)

	Argh. Progress, probably, but keeping all the invariants with two
	differently over-allocated lists is a bit of a mess.

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -103,8 +103,9 @@
                                               lltype.Signed, lltype.Signed],
                                              lltype.Signed))
     LOOKCLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
+                                                 lltype.Signed,
                                                  lltype.Signed],
-                                                lltype.Signed))
+                                                lltype.Void))
 
     fields =          [ ("num_items", lltype.Signed),
                         ("num_used_items", lltype.Signed),
@@ -404,27 +405,35 @@
                                            lltype.malloc(DICTINDEX_BYTE.TO, n,
                                                          zero=True))
         d.lookup_function = DICT.byte_lookup_function
-        return DICT.byte_lookup_clean_function
     elif n <= 65536:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_SHORT.TO, n,
                                                          zero=True))
         d.lookup_function = DICT.short_lookup_function
-        return DICT.short_lookup_clean_function
     elif IS_64BIT and n <= 2 ** 32:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_INT.TO, n,
                                                          zero=True))
         d.lookup_function = DICT.int_lookup_function
-        return DICT.int_lookup_clean_function
     else:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_LONG.TO, n,
                                                          zero=True))
         d.lookup_function = DICT.long_lookup_function
-        return DICT.long_lookup_clean_function
 ll_malloc_indexes_and_choose_lookup._always_inline_ = True
 
+def ll_pick_insert_clean_function(d):
+    DICT = lltype.typeOf(d).TO
+    if d.lookup_function == DICT.byte_lookup_function:
+        return d.byte_lookup_clean_function
+    if d.lookup_function == DICT.short_lookup_function:
+        return d.short_lookup_clean_function
+    if IS_64BIT and d.lookup_function == DICT.int_lookup_function:
+        return d.int_lookup_clean_function
+    if d.lookup_function == DICT.long_lookup_function:
+        return d.long_lookup_clean_function
+    assert False
+
 def ll_valid_from_flag(entries, i):
     return entries[i].f_valid
 
@@ -487,7 +496,7 @@
 
 # It may be safe to look inside always, it has a few branches though, and their
 # frequencies needs to be investigated.
- at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
+#@jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     ENTRY = lltype.typeOf(d.entries).TO.OF
     if i >= 0:
@@ -495,7 +504,9 @@
         entry.value = value
     else:
         if len(d.entries) == d.num_used_items:
-            ll_dict_grow(d)
+            if ll_dict_grow(d):
+                insertcleanfn = ll_pick_insert_clean_function(d)
+                insertcleanfn(d, hash, d.num_used_items)
         entry = d.entries[d.num_used_items]
         entry.key = key
         entry.value = value
@@ -512,22 +523,40 @@
             ll_assert(rc > 0, "ll_dict_resize failed?")
         d.resize_counter = rc
 
-def ll_dict_grow(d):
-    if d.num_items < d.num_used_items // 4:
-        xxxxxxxxx
+def _ll_len_of_d_indexes(d):
+    # xxx Haaaack: returns len(d.indexes).  Works independently of
+    # the exact type pointed to by d, using a forced cast...
+    return len(rffi.cast(DICTINDEX_BYTE, d.indexes))
+
+def _overallocate_entries_len(baselen):
     # This over-allocates proportional to the list size, making room
     # for additional growth.  The over-allocation is mild, but is
     # enough to give linear-time amortized behavior over a long
     # sequence of appends() in the presence of a poorly-performing
     # system malloc().
     # The growth pattern is:  0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
-    newsize = len(d.entries) + 1
+    newsize = baselen + 1
     if newsize < 9:
         some = 3
     else:
         some = 6
     some += newsize >> 3
-    new_allocated = newsize + some
+    return newsize + some
+
+def ll_dict_grow(d):
+    if d.num_items < d.num_used_items // 4:
+        ll_dict_remove_deleted_items(d)
+        return True
+
+    new_allocated = _overallocate_entries_len(len(d.entries))
+
+    # Detect an obscure case where the indexes numeric type is too
+    # small to store all the entry indexes
+    if (max(128, _ll_len_of_d_indexes(d)) - new_allocated
+                   < MIN_INDEXES_MINUS_ENTRIES):
+        ll_dict_remove_deleted_items(d)
+        return True
+
     newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
     #
     # XXX we should do this with rgc.ll_arraycopy()!!
@@ -544,8 +573,40 @@
             dst.f_valid = src.f_valid
         i += 1
     d.entries = newitems
+    return False
+
+def ll_dict_remove_deleted_items(d):
+    new_allocated = _overallocate_entries_len(d.num_items)
+    if new_allocated < len(d.entries) // 2:
+        newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+    else:
+        newitems = d.entries
+    #
+    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
+    isrc = 0
+    idst = 0
+    while isrc < len(d.entries):
+        if d.entries.valid(isrc):
+            src = d.entries[isrc]
+            dst = newitems[idst]
+            dst.key = src.key
+            dst.value = src.value
+            if hasattr(ENTRY, 'f_hash'):
+                dst.f_hash = src.f_hash
+            if hasattr(ENTRY, 'f_valid'):
+                assert src.f_valid
+                dst.f_valid = True
+            idst += 1
+        isrc += 1
+    d.entries = newitems
+    assert d.num_items == idst
+    d.num_used_items = idst
+
+    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+
 
 def ll_dict_insertclean(d, key, value, hash, lookcleanfn):
+    XXXXXXX
     # Internal routine used by ll_dict_resize() to insert an item which is
     # known to be absent from the dict.  This routine also assumes that
     # the dict contains no deleted entries.  This routine has the advantage
@@ -598,7 +659,7 @@
     # make a 'new_size' estimate and shrink it if there are many
     # deleted entry markers.  See CPython for why it is a good idea to
     # quadruple the dictionary size as long as it's not too big.
-    num_items = d.num_used_items
+    num_items = d.num_items
     if num_items > 50000:
         new_estimate = num_items * 2
     else:
@@ -606,21 +667,27 @@
     new_size = DICT_INITSIZE
     while new_size <= new_estimate:
         new_size *= 2
-    lookcleanfn = ll_malloc_indexes_and_choose_lookup(d, new_size)
-    d.num_items = 0
-    d.num_used_items = 0
-    d.resize_counter = new_size * 2
+
+    if new_size < _ll_len_of_d_indexes(d):
+        ll_dict_remove_deleted_items(d)
+    else:
+        ll_dict_reindex(d, new_size)
+ll_dict_resize.oopspec = 'dict.resize(d)'
+
+def ll_dict_reindex(d, new_size):
+    ll_malloc_indexes_and_choose_lookup(d, new_size)
+    d.resize_counter = new_size * 2 - d.num_items * 3
+    assert d.resize_counter > 0
     #
+    insertcleanfn = ll_pick_insert_clean_function(d)
     entries = d.entries
     i = 0
-    while i < num_items:
+    while i < d.num_used_items:
         if entries.valid(i):
             hash = entries.hash(i)
-            entry = entries[i]
-            ll_dict_insertclean(d, entry.key, entry.value, hash, lookcleanfn)
+            insertcleanfn(d, hash, i)
         i += 1
     #old_entries.delete() XXXX!
-ll_dict_resize.oopspec = 'dict.resize(d)'
 
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
@@ -628,6 +695,7 @@
 FREE = 0
 DELETED = 1
 VALID_OFFSET = 2
+MIN_INDEXES_MINUS_ENTRIES = VALID_OFFSET + 1
 
 FLAG_LOOKUP = 0
 FLAG_STORE = 1
@@ -731,7 +799,7 @@
                 deletedslot = i
             perturb >>= PERTURB_SHIFT
 
-    def ll_dict_lookup_clean(d, hash):
+    def ll_dict_store_clean(d, hash, index):
         # a simplified version of ll_dict_lookup() which assumes that the
         # key is new, and the dictionary doesn't contain deleted entries.
         # It only finds the next free slot for the given hash.
@@ -743,12 +811,10 @@
             i = (i << 2) + i + perturb + 1
             i = i & mask
             perturb >>= PERTURB_SHIFT
-        index = d.num_used_items
         indexes[i] = rffi.cast(T, index + VALID_OFFSET)
-        return index
 
     return (llhelper(LOOKUP_FUNC, ll_dict_lookup),
-            llhelper(LOOKCLEAN_FUNC, ll_dict_lookup_clean))
+            llhelper(LOOKCLEAN_FUNC, ll_dict_store_clean))
 
 # ____________________________________________________________
 #
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -60,6 +60,30 @@
         rdict.ll_dict_setitem(ll_d, llstr("abc"), 43)
         assert rdict.ll_dict_getitem(ll_d, lls) == 43
 
+    def test_dict_store_get(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        for i in range(20):
+            for j in range(i):
+                assert rdict.ll_dict_getitem(ll_d, llstr(str(j))) == j
+            rdict.ll_dict_setitem(ll_d, llstr(str(i)), i)
+        assert ll_d.num_items == 20
+        for i in range(20):
+            assert rdict.ll_dict_getitem(ll_d, llstr(str(i))) == i
+
+    def test_dict_store_get_del(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        for i in range(20):
+            for j in range(0, i, 2):
+                assert rdict.ll_dict_getitem(ll_d, llstr(str(j))) == j
+            rdict.ll_dict_setitem(ll_d, llstr(str(i)), i)
+            if i % 2 != 0:
+                rdict.ll_dict_delitem(ll_d, llstr(str(i)))
+        assert ll_d.num_items == 10
+        for i in range(0, 20, 2):
+            assert rdict.ll_dict_getitem(ll_d, llstr(str(i))) == i
+
     def test_dict_del_lastitem(self):
         DICT = self._get_str_dict()
         ll_d = rdict.ll_newdict(DICT)
@@ -94,6 +118,15 @@
         for item in ['a', 'b', 'c', 'd', 'e', 'f']:
             assert rdict.ll_dict_getitem(ll_d, llstr(item)) == ord(item) - ord('a') + 1
 
+    def test_dict_grow_cleanup(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        lls = llstr("a")
+        for i in range(20):
+            rdict.ll_dict_setitem(ll_d, lls, i)
+            rdict.ll_dict_delitem(ll_d, lls)
+        assert ll_d.num_used_items <= 4
+
     def test_dict_iteration(self):
         DICT = self._get_str_dict()
         ll_d = rdict.ll_newdict(DICT)

From noreply at buildbot.pypy.org  Wed Oct  9 18:44:43 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 18:44:43 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal,
 arigo) simple fixes to make iteration work
Message-ID: <20131009164443.B421B1C359C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67253:876473158bd8
Date: 2013-10-09 18:43 +0200
http://bitbucket.org/pypy/pypy/changeset/876473158bd8/

Log:	(fijal, arigo) simple fixes to make iteration work

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -878,14 +878,17 @@
 #
 #  Iteration.
 
+def get_ll_dictiter(DICTPTR):
+    return lltype.Ptr(lltype.GcStruct('dictiter',
+                                      ('dict', DICTPTR),
+                                      ('index', lltype.Signed)))
+
 class DictIteratorRepr(AbstractDictIteratorRepr):
 
     def __init__(self, r_dict, variant="keys"):
         self.r_dict = r_dict
         self.variant = variant
-        self.lowleveltype = lltype.Ptr(lltype.GcStruct('dictiter',
-                                         ('dict', r_dict.lowleveltype),
-                                         ('index', lltype.Signed)))
+        self.lowleveltype = get_ll_dictiter(r_dict.lowleveltype)
         self.ll_dictiter = ll_dictiter
         self.ll_dictnext = ll_dictnext_group[variant]
 
@@ -905,31 +908,35 @@
     def ll_dictnext(RETURNTYPE, iter):
         # note that RETURNTYPE is None for keys and values
         dict = iter.dict
-        if dict:
-            entries = dict.entries
-            index = iter.index
-            assert index >= 0
-            entries_len = len(entries)
-            while index < entries_len:
-                entry = entries[index]
-                is_valid = entries.valid(index)
-                index = index + 1
-                if is_valid:
-                    iter.index = index
-                    if RETURNTYPE is lltype.Void:
-                        return None
-                    elif kind == 'items':
-                        r = lltype.malloc(RETURNTYPE.TO)
-                        r.item0 = recast(RETURNTYPE.TO.item0, entry.key)
-                        r.item1 = recast(RETURNTYPE.TO.item1, entry.value)
-                        return r
-                    elif kind == 'keys':
-                        return entry.key
-                    elif kind == 'values':
-                        return entry.value
-            # clear the reference to the dict and prevent restarts
-            iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
+        if not dict:
+            raise StopIteration
+
+        entries = dict.entries
+        index = iter.index
+        assert index >= 0
+        entries_len = dict.num_used_items
+        while index < entries_len:
+            entry = entries[index]
+            is_valid = entries.valid(index)
+            index = index + 1
+            if is_valid:
+                iter.index = index
+                if RETURNTYPE is lltype.Void:
+                    return None
+                elif kind == 'items':
+                    r = lltype.malloc(RETURNTYPE.TO)
+                    r.item0 = recast(RETURNTYPE.TO.item0, entry.key)
+                    r.item1 = recast(RETURNTYPE.TO.item1, entry.value)
+                    return r
+                elif kind == 'keys':
+                    return entry.key
+                elif kind == 'values':
+                    return entry.value
+                
+        # clear the reference to the dict and prevent restarts
+        iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
         raise StopIteration
+
     return ll_dictnext
 
 ll_dictnext_group = {'keys'  : _make_ll_dictnext('keys'),

From noreply at buildbot.pypy.org  Wed Oct  9 19:12:48 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 19:12:48 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal,
 arigo) remove the mess from popitem that stored stuff in a global
Message-ID: <20131009171248.C28831C33FE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67254:c6c3940285e7
Date: 2013-10-09 18:53 +0200
http://bitbucket.org/pypy/pypy/changeset/c6c3940285e7/

Log:	(fijal, arigo) remove the mess from popitem that stored stuff in a
	global structure.

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -932,7 +932,7 @@
                     return entry.key
                 elif kind == 'values':
                     return entry.value
-                
+
         # clear the reference to the dict and prevent restarts
         iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
         raise StopIteration
@@ -1058,32 +1058,20 @@
     i = ll_dict_lookup(d, key, d.keyhash(key))
     return not i & HIGHEST_BIT
 
-POPITEMINDEX = lltype.Struct('PopItemIndex', ('nextindex', lltype.Signed))
-global_popitem_index = lltype.malloc(POPITEMINDEX, zero=True, immortal=True)
+def _ll_getnextitem(dic):
+    if dic.num_items == 0:
+        raise KeyError
 
-def _ll_getnextitem(dic):
     entries = dic.entries
-    ENTRY = lltype.typeOf(entries).TO.OF
-    dmask = len(entries) - 1
-    if hasattr(ENTRY, 'f_hash'):
-        if entries.valid(0):
-            return 0
-        base = entries[0].f_hash
-    else:
-        base = global_popitem_index.nextindex
-    counter = 0
-    while counter <= dmask:
-        i = (base + counter) & dmask
-        counter += 1
+
+    i = dic.num_used_items - 1
+    while True:
         if entries.valid(i):
             break
-    else:
-        raise KeyError
-    if hasattr(ENTRY, 'f_hash'):
-        entries[0].f_hash = base + counter
-    else:
-        global_popitem_index.nextindex = base + counter
-    return i
+        i -= 1
+
+    key = entries[i].key
+    return dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
 
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -146,11 +146,15 @@
         ll_d = rdict.ll_newdict(DICT)
         rdict.ll_dict_setitem(ll_d, llstr("k"), 1)
         rdict.ll_dict_setitem(ll_d, llstr("j"), 2)
-        ll_elem = rdict.ll_popitem(lltype.Ptr(
-            lltype.GcStruct('x', ('item0', lltype.Ptr(rstr.STR)),
-                            ('item1', lltype.Signed))), ll_d)
+        TUP = lltype.Ptr(lltype.GcStruct('x', ('item0', lltype.Ptr(rstr.STR)),
+                                              ('item1', lltype.Signed)))
+        ll_elem = rdict.ll_popitem(TUP, ll_d)
         assert hlstr(ll_elem.item0) == "j"
         assert ll_elem.item1 == 2
+        ll_elem = rdict.ll_popitem(TUP, ll_d)
+        assert hlstr(ll_elem.item0) == "k"
+        assert ll_elem.item1 == 1
+        py.test.raises(KeyError, rdict.ll_popitem, TUP, ll_d)
 
     def test_direct_enter_and_del(self):
         def eq(a, b):

From noreply at buildbot.pypy.org  Wed Oct  9 19:13:15 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 19:13:15 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (arigo) add some more
	paranoia
Message-ID: <20131009171315.DA6E41C33FE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67255:775d6e5f7190
Date: 2013-10-09 19:12 +0200
http://bitbucket.org/pypy/pypy/changeset/775d6e5f7190/

Log:	(arigo) add some more paranoia

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -700,10 +700,20 @@
 FLAG_LOOKUP = 0
 FLAG_STORE = 1
 FLAG_DELETE = 2
+FLAG_DELETE_TRY_HARD = 3
 
 def new_lookup_functions(LOOKUP_FUNC, LOOKCLEAN_FUNC, T):
     INDEXES = lltype.Ptr(lltype.GcArray(T))
 
+    def ll_kill_something(d):
+        i = 0
+        while True:
+            index = rffi.cast(lltype.Signed, d.indexes[i])
+            if index >= VALID_OFFSET:
+                d.indexes[i] = rffi.cast(T, DELETED)
+                return index
+            i += 1
+
     @jit.look_inside_iff(lambda d, key, hash, store_flag:
                          jit.isvirtual(d) and jit.isconstant(key))
     def ll_dict_lookup(d, key, hash, store_flag):
@@ -749,6 +759,8 @@
             # pristine entry -- lookup failed
             if store_flag == FLAG_STORE:
                 indexes[i] = rffi.cast(T, d.num_used_items + VALID_OFFSET)
+            elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
+                return ll_kill_something(d)
             return -1
 
         # In the loop, a deleted entry (everused and not valid) is by far
@@ -767,6 +779,8 @@
                         deletedslot = i
                     indexes[deletedslot] = rffi.cast(T, d.num_used_items +
                                                      VALID_OFFSET)
+                elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
+                    return ll_kill_something(d)
                 return -1
             elif index >= VALID_OFFSET:
                 checkingkey = entries[index].key
@@ -1071,7 +1085,15 @@
         i -= 1
 
     key = entries[i].key
-    return dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    index = dic.lookup_function(dic, key, dic.keyhash(key),
+                                FLAG_DELETE_TRY_HARD)
+    # if the lookup function returned me a random strange thing,
+    # don't care about deleting the item
+    if index == dic.num_used_items - 1:
+        dic.num_used_items -= 1
+    else:
+        assert index != -1
+    return index
 
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -170,7 +170,7 @@
             rdict.ll_dict_setitem(ll_d, num, 1)
             rdict.ll_dict_delitem(ll_d, num)
             for k in foreach_index(ll_d):
-                assert k < 0
+                assert k < rdict.VALID_OFFSET
 
 class TestRdict(BaseRtypingTest):
 

From noreply at buildbot.pypy.org  Wed Oct  9 19:17:59 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  9 Oct 2013 19:17:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: some tests for error messages
Message-ID: <20131009171759.558D91C33FE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67256:d91e484f733e
Date: 2013-10-09 19:17 +0200
http://bitbucket.org/pypy/pypy/changeset/d91e484f733e/

Log:	some tests for error messages

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3442,6 +3442,29 @@
 
         a.build_types(f, [str])
 
+    def test_negative_number_find(self):
+        def f(s, e):
+            return "xyz".find("x", s, e)
+
+        a = self.RPythonAnnotator()
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        def f(s, e):
+            return "xyz".rfind("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+
+        def f(s, e):
+            return "xyz".count("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        
+
     def test_setslice(self):
         def f():
             lst = [2, 5, 7]
@@ -4080,7 +4103,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify instances with no common base class" 
+        assert ("RPython cannot unify instances with no common base class"
                 in exc.value.msg)
 
     def test_unionerror_iters(self):
@@ -4096,7 +4119,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify incompatible iterator variants" in 
+        assert ("RPython cannot unify incompatible iterator variants" in
                 exc.value.msg)
 
     def test_variable_getattr(self):

From noreply at buildbot.pypy.org  Wed Oct  9 19:22:42 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  9 Oct 2013 19:22:42 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal,
	arigo) RPythonification
Message-ID: <20131009172242.1320A1C33FE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67257:2a1d6345670e
Date: 2013-10-09 19:21 +0200
http://bitbucket.org/pypy/pypy/changeset/2a1d6345670e/

Log:	(fijal, arigo) RPythonification

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -139,8 +139,9 @@
     adtmeths['KEY']   = DICTKEY
     adtmeths['VALUE'] = DICTVALUE
     adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
-    adtmeths['empty_array'] = DICTENTRYARRAY.allocate(0)
 
+    family = LookupFamily()
+    family.empty_array = DICTENTRYARRAY.allocate(0)
     for name, T in [('byte', rffi.UCHAR),
                     ('short', rffi.USHORT),
                     ('int', rffi.UINT),
@@ -149,13 +150,19 @@
             continue
         lookupfn, lookcleanfn = new_lookup_functions(LOOKUP_FUNC,
                                                      LOOKCLEAN_FUNC, T=T)
-        adtmeths['%s_lookup_function' % name] = lookupfn
-        adtmeths['%s_lookup_clean_function' % name] = lookcleanfn
+        setattr(family, '%s_lookup_function' % name, lookupfn)
+        setattr(family, '%s_insert_clean_function' % name, lookcleanfn)
+    adtmeths['lookup_family'] = family
 
     DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
                                 *fields))
     return DICT
 
+class LookupFamily:
+    def _freeze_(self):
+        return True
+
+
 class DictRepr(AbstractDictRepr):
 
     def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
@@ -404,34 +411,35 @@
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_BYTE.TO, n,
                                                          zero=True))
-        d.lookup_function = DICT.byte_lookup_function
+        d.lookup_function = DICT.lookup_family.byte_lookup_function
     elif n <= 65536:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_SHORT.TO, n,
                                                          zero=True))
-        d.lookup_function = DICT.short_lookup_function
+        d.lookup_function = DICT.lookup_family.short_lookup_function
     elif IS_64BIT and n <= 2 ** 32:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_INT.TO, n,
                                                          zero=True))
-        d.lookup_function = DICT.int_lookup_function
+        d.lookup_function = DICT.lookup_family.int_lookup_function
     else:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_LONG.TO, n,
                                                          zero=True))
-        d.lookup_function = DICT.long_lookup_function
+        d.lookup_function = DICT.lookup_family.long_lookup_function
 ll_malloc_indexes_and_choose_lookup._always_inline_ = True
 
 def ll_pick_insert_clean_function(d):
     DICT = lltype.typeOf(d).TO
-    if d.lookup_function == DICT.byte_lookup_function:
-        return d.byte_lookup_clean_function
-    if d.lookup_function == DICT.short_lookup_function:
-        return d.short_lookup_clean_function
-    if IS_64BIT and d.lookup_function == DICT.int_lookup_function:
-        return d.int_lookup_clean_function
-    if d.lookup_function == DICT.long_lookup_function:
-        return d.long_lookup_clean_function
+    if d.lookup_function == DICT.lookup_family.byte_lookup_function:
+        return DICT.lookup_family.byte_insert_clean_function
+    if d.lookup_function == DICT.lookup_family.short_lookup_function:
+        return DICT.lookup_family.short_insert_clean_function
+    if IS_64BIT:
+        if d.lookup_function == DICT.lookup_family.int_lookup_function:
+            return DICT.lookup_family.int_insert_clean_function
+    if d.lookup_function == DICT.lookup_family.long_lookup_function:
+        return DICT.lookup_family.long_insert_clean_function
     assert False
 
 def ll_valid_from_flag(entries, i):
@@ -838,7 +846,7 @@
 
 def ll_newdict(DICT):
     d = DICT.allocate()
-    d.entries = DICT.empty_array
+    d.entries = DICT.lookup_family.empty_array
     ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
     d.num_items = 0
     d.num_used_items = 0

From noreply at buildbot.pypy.org  Wed Oct  9 19:27:34 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  9 Oct 2013 19:27:34 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal, arigo)
Message-ID: <20131009172734.27D2D1C3526@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67258:f0f77aef2ee3
Date: 2013-10-09 19:26 +0200
http://bitbucket.org/pypy/pypy/changeset/f0f77aef2ee3/

Log:	(fijal, arigo) More RPythonification

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -448,12 +448,12 @@
 def ll_valid_from_key(entries, i):
     ENTRIES = lltype.typeOf(entries).TO
     dummy = ENTRIES.dummy_obj.ll_dummy_value
-    return entries.everused(i) and entries[i].key != dummy
+    return entries[i].key != dummy
 
 def ll_valid_from_value(entries, i):
     ENTRIES = lltype.typeOf(entries).TO
     dummy = ENTRIES.dummy_obj.ll_dummy_value
-    return entries.everused(i) and entries[i].value != dummy
+    return entries[i].value != dummy
 
 def ll_mark_deleted_in_flag(entries, i):
     entries[i].f_valid = False
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -39,12 +39,17 @@
     return c
 
 class TestRDictDirect(object):
+    dummykeyobj = None
+    dummyvalueobj = None
+
     def _get_str_dict(self):
         # STR -> lltype.Signed
         DICT = rdict.get_ll_dict(lltype.Ptr(rstr.STR), lltype.Signed,
                                  ll_fasthash_function=rstr.LLHelpers.ll_strhash,
                                  ll_hash_function=rstr.LLHelpers.ll_strhash,
-                                 ll_eq_function=rstr.LLHelpers.ll_streq)
+                                 ll_eq_function=rstr.LLHelpers.ll_streq,
+                                 dummykeyobj=self.dummykeyobj,
+                                 dummyvalueobj=self.dummyvalueobj)
         return DICT
 
     def test_dict_creation(self):
@@ -172,6 +177,16 @@
             for k in foreach_index(ll_d):
                 assert k < rdict.VALID_OFFSET
 
+
+class TestRDictDirectDummyKey(TestRDictDirect):
+    class dummykeyobj:
+        ll_dummy_value = llstr("dupa")
+
+class TestRDictDirectDummyValue(TestRDictDirect):
+    class dummyvalueobj:
+        ll_dummy_value = -42
+
+
 class TestRdict(BaseRtypingTest):
 
     def test_dict_creation(self):

From noreply at buildbot.pypy.org  Wed Oct  9 21:57:44 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed,  9 Oct 2013 21:57:44 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: bah,
	forgot to hg add this file
Message-ID: <20131009195744.946031C359C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67259:4337806d2cd5
Date: 2013-10-09 21:56 +0200
http://bitbucket.org/pypy/pypy/changeset/4337806d2cd5/

Log:	bah, forgot to hg add this file

diff --git a/pypy/module/_cffi_backend/test/test_extra.py b/pypy/module/_cffi_backend/test/test_extra.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/test/test_extra.py
@@ -0,0 +1,36 @@
+# side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule()
+from pypy.module._cffi_backend import misc
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+
+class AppTestFastPath(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def setup_method(self, meth):
+        def forbidden(self, *args):
+            assert False, 'The slow path is forbidden'
+        self._original = W_CTypePtrOrArray._convert_array_from_listview.im_func
+        W_CTypePtrOrArray._convert_array_from_listview = forbidden
+
+    def teardown_method(self, meth):
+        W_CTypePtrOrArray._convert_array_from_listview = self._original
+
+    def test_fast_init_from_list(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, None)
+        buf = _cffi_backend.newp(LONG_ARRAY, [1, 2, 3])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == 3
+
+    def test_fast_init_from_list_float(self):
+        import _cffi_backend
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        P_DOUBLE = _cffi_backend.new_pointer_type(DOUBLE)
+        DOUBLE_ARRAY = _cffi_backend.new_array_type(P_DOUBLE, None)
+        buf = _cffi_backend.newp(DOUBLE_ARRAY, [1.1, 2.2, 3.3])
+        assert buf[0] == 1.1
+        assert buf[1] == 2.2
+        assert buf[2] == 3.3
+

From noreply at buildbot.pypy.org  Thu Oct 10 00:06:28 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 00:06:28 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-sort: add tests from numpy,
	fix possible segfault
Message-ID: <20131009220628.162B01C359C@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-sort
Changeset: r67260:307fed969929
Date: 2013-10-07 23:38 +0300
http://bitbucket.org/pypy/pypy/changeset/307fed969929/

Log:	add tests from numpy, fix possible segfault

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2652,55 +2652,6 @@
         assert array([1, 2, 3], '>i2')[::2].tostring() == '\x00\x01\x00\x03'
         assert array(0, dtype='i2').tostring() == '\x00\x00'
 
-    def test_argsort_dtypes(self):
-        from numpypy import array, arange
-        assert array(2.0).argsort() == 0
-        nnp = self.non_native_prefix
-        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
-                        nnp + 'i2', complex]:
-            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
-            c = a.copy()
-            res = a.argsort()
-            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
-                'a,res,dtype %r,%r,%r' % (a,res,dtype)
-            assert (a == c).all() # not modified
-            a = arange(100)
-            assert (a.argsort() == a).all()
-        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
-
-    def test_argsort_nd(self):
-        from numpypy import array
-        a = array([[4, 2], [1, 3]])
-        assert (a.argsort() == [[1, 0], [0, 1]]).all()
-        a = array(range(10) + range(10) + range(10))
-        b = a.argsort()
-        assert (b[:3] == [0, 10, 20]).all()
-        #trigger timsort 'run' mode which calls arg_getitem_slice
-        a = array(range(100) + range(100) + range(100))
-        b = a.argsort()
-        assert (b[:3] == [0, 100, 200]).all()
-        a = array([[[]]]).reshape(3,4,0)
-        b = a.argsort()
-        assert b.size == 0
-
-    def test_argsort_random(self):
-        from numpypy import array
-        from _random import Random
-        rnd = Random(1)
-        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
-        a.argsort()
-
-    def test_argsort_axis(self):
-        from numpypy import array
-        a = array([[4, 2], [1, 3]])
-        assert (a.argsort(axis=None) == [2, 1, 3, 0]).all()
-        assert (a.argsort(axis=-1) == [[1, 0], [0, 1]]).all()
-        assert (a.argsort(axis=0) == [[1, 0], [0, 1]]).all()
-        assert (a.argsort(axis=1) == [[1, 0], [0, 1]]).all()
-        a = array([[3, 2, 1], [1, 2, 3]])
-        assert (a.argsort(axis=0) == [[1, 0, 0], [0, 1, 1]]).all()
-        assert (a.argsort(axis=1) == [[2, 1, 0], [0, 1, 2]]).all()
-
 
 class AppTestRanges(BaseNumpyAppTest):
     def test_arange(self):
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -0,0 +1,283 @@
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestSupport(BaseNumpyAppTest):
+    def setup_class(cls):
+        import struct
+        BaseNumpyAppTest.setup_class.im_func(cls)
+        cls.w_data = cls.space.wrap(struct.pack('dddd', 1, 2, 3, 4))
+        cls.w_fdata = cls.space.wrap(struct.pack('f', 2.3))
+        cls.w_float16val = cls.space.wrap('\x00E') # 5.0 in float16
+        cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
+        cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
+        cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+
+    def test_argsort_dtypes(self):
+        from numpypy import array, arange
+        assert array(2.0).argsort() == 0
+        nnp = self.non_native_prefix
+        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
+                        nnp + 'i2', complex]:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            c = a.copy()
+            res = a.argsort()
+            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
+                'a,res,dtype %r,%r,%r' % (a,res,dtype)
+            assert (a == c).all() # not modified
+            a = arange(100)
+            assert (a.argsort() == a).all()
+        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
+
+    def test_argsort_nd(self):
+        from numpypy import array
+        a = array([[4, 2], [1, 3]])
+        assert (a.argsort() == [[1, 0], [0, 1]]).all()
+        a = array(range(10) + range(10) + range(10))
+        b = a.argsort()
+        assert (b[:3] == [0, 10, 20]).all()
+        #trigger timsort 'run' mode which calls arg_getitem_slice
+        a = array(range(100) + range(100) + range(100))
+        b = a.argsort()
+        assert (b[:3] == [0, 100, 200]).all()
+        a = array([[[]]]).reshape(3,4,0)
+        b = a.argsort()
+        assert b.size == 0
+
+    def test_argsort_random(self):
+        from numpypy import array
+        from _random import Random
+        rnd = Random(1)
+        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
+        a.argsort()
+
+    def test_argsort_axis(self):
+        from numpypy import array
+        a = array([[4, 2], [1, 3]])
+        assert (a.argsort(axis=None) == [2, 1, 3, 0]).all()
+        assert (a.argsort(axis=-1) == [[1, 0], [0, 1]]).all()
+        assert (a.argsort(axis=0) == [[1, 0], [0, 1]]).all()
+        assert (a.argsort(axis=1) == [[1, 0], [0, 1]]).all()
+        a = array([[3, 2, 1], [1, 2, 3]])
+        assert (a.argsort(axis=0) == [[1, 0, 0], [0, 1, 1]]).all()
+        assert (a.argsort(axis=1) == [[2, 1, 0], [0, 1, 2]]).all()
+
+# tests from numpy/tests/test_multiarray.py
+    def test_sort(self):
+        # test ordering for floats and complex containing nans. It is only
+        # necessary to check the lessthan comparison, so sorts that
+        # only follow the insertion sort path are sufficient. We only
+        # test doubles and complex doubles as the logic is the same.
+
+        # check doubles
+        from numpypy import array, nan, zeros, complex128, arange, empty, dtype
+        msg = "Test real sort order with nans"
+        a = array([nan, 1, 0])
+        b = a.sort()
+        assert (b == a[::-1]).all(), msg
+        # check complex
+        msg = "Test complex sort order with nans"
+        a = zeros(9, dtype=complex128)
+        a.real += [nan, nan, nan, 1, 0, 1, 1, 0, 0]
+        a.imag += [nan, 1, 0, nan, nan, 1, 0, 1, 0]
+        b = a.sort()
+        assert (b == a[::-1]).all(), msg
+
+        # all c scalar sorts use the same code with different types
+        # so it suffices to run a quick check with one type. The number
+        # of sorted items must be greater than ~50 to check the actual
+        # algorithm because quick and merge sort fall over to insertion
+        # sort for small arrays.
+        a = arange(101)
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h'] :
+            msg = "scalar sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            self.assert_equal(c, a, msg)
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+        # test complex sorts. These use the same code as the scalars
+        # but the compare fuction differs.
+        ai = a*1j + 1
+        bi = b*1j + 1
+        for kind in ['q', 'm', 'h'] :
+            msg = "complex sort, real part == 1, kind=%s" % kind
+            c = ai.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+            c = bi.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+        ai = a + 1j
+        bi = b + 1j
+        for kind in ['q', 'm', 'h'] :
+            msg = "complex sort, imag part == 1, kind=%s" % kind
+            c = ai.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+            c = bi.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+
+        # test string sorts.
+        s = 'aaaaaaaa'
+        a = array([s + chr(i) for i in range(101)])
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h'] :
+            msg = "string sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+        # check axis handling. This should be the same for all type
+        # specific sorts, so we only check it for one type and one kind
+        a = array([[3, 2], [1, 0]])
+        b = array([[1, 0], [3, 2]])
+        c = array([[2, 3], [0, 1]])
+        d = a.copy()
+        d.sort(axis=0)
+        assert (d == b).all(), "test sort with axis=0"
+        d = a.copy()
+        d.sort(axis=1)
+        assert (d == c).all(), "test sort with axis=1"
+        d = a.copy()
+        d.sort()
+        assert (d == c).all(), "test sort with default axis"
+
+
+        # test record array sorts.
+        dt =dtype([('f', float), ('i', int)])
+        a = array([(i, i) for i in range(101)], dtype = dt)
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "object sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_unicode(self):
+        from numpypy import array
+        # test unicode sorts.
+        s = 'aaaaaaaa'
+        try:
+            a = array([s + chr(i) for i in range(101)], dtype=unicode)
+            b = a[::-1].copy()
+        except:
+            skip('unicode type not supported yet')
+        for kind in ['q', 'm', 'h'] :
+            msg = "unicode sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_objects(self):
+        # test object array sorts.
+        from numpypy import empty
+        try:
+            a = empty((101,), dtype=object)
+        except:
+            skip('object type not supported yet')
+        a[:] = list(range(101))
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "object sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_datetime(self):
+        from numpypy import arange
+        # test datetime64 sorts.
+        try:
+            a = arange(0, 101, dtype='datetime64[D]')
+        except:
+            skip('datetime type not supported yet')
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "datetime64 sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+        # test timedelta64 sorts.
+        a = arange(0, 101, dtype='timedelta64[D]')
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "timedelta64 sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_order(self):
+        from numpypy import array
+        from sys import byteorder
+        # Test sorting an array with fields
+        x1 = array([21, 32, 14])
+        x2 = array(['my', 'first', 'name'])
+        x3=array([3.1, 4.5, 6.2])
+        r=array([x1, x2, x3], dtype=[('id','i'),('word','S'),('number','f')])
+
+        r.sort(order=['id'])
+        assert (r['id'] == [14, 21, 32]).all()
+        assert (r['word'] == ['name', 'my', 'first']).all()
+        assert (r['number'] == [6.2, 3.1, 4.5]).all()
+
+        r.sort(order=['word'])
+        assert (r['id'] == [32, 21, 14]).all()
+        assert (r['word'] == ['first', 'my', 'name']).all()
+        assert (r['number'] == [4.5, 3.1, 6.2]).all()
+
+        r.sort(order=['number'])
+        assert (r['id'] == [21, 32, 14]).all()
+        assert (r['word'] == ['my', 'first', 'name']).all()
+        assert (r['number'] == [3.1, 4.5, 6.2]).all()
+
+        if byteorder == 'little':
+            strtype = '>i2'
+        else:
+            strtype = '<i2'
+        mydtype = [('name', 'S5'), ('col2', strtype)]
+        r = array([('a', 1), ('b', 255), ('c', 3), ('d', 258)],
+                     dtype= mydtype)
+        r.sort(order='col2')
+        assert (r['col2'] == [1, 3, 255, 258]).all()
+        assert (r == array([('a', 1), ('c', 3), ('b', 255), ('d', 258)],
+                                 dtype=mydtype)).all()
+
+
+
+
+# tests from numpy/tests/test_regression.py
+    def test_sort_bigendian(self):
+        from numpypy import array, dtype
+        a = array(range(11),dtype='float64')
+        c = a.astype(dtype('<f8'))
+        c.sort()
+        assert (a-c).abs().max() < 1e-32
+
+    def test_string_sort_with_zeros(self):
+        from numpypy import fromstring
+        """Check sort for strings containing zeros."""
+        x = fromstring("\x00\x02\x00\x01", dtype="S2")
+        y = fromstring("\x00\x01\x00\x02", dtype="S2")
+        assert all(x.sort(kind="q") == y)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1864,7 +1864,6 @@
         return interp_boxes.W_StringBox(arr,  0, arr.dtype)
 
     def fill(self, storage, width, box, start, stop, offset):
-        from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArrayNotOwning
         for i in xrange(start, stop, width):
             self._store(storage, i, offset, box)
 
@@ -1873,6 +1872,13 @@
 class UnicodeType(BaseType, BaseStringType):
     T = lltype.UniChar
 
+    @jit.unroll_safe
+    def coerce(self, space, dtype, w_item):
+        if isinstance(w_item, interp_boxes.W_UnicodeBox):
+            return w_item
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "coerce (probably from set_item) not implemented for unicode type"))
+
 NonNativeUnicodeType = UnicodeType
 
 class VoidType(BaseType, BaseStringType):

From noreply at buildbot.pypy.org  Thu Oct 10 00:06:29 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 00:06:29 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-sort: fix tests for -A
Message-ID: <20131009220629.5AC7F1C359C@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-sort
Changeset: r67261:c710232971f1
Date: 2013-10-08 00:21 +0300
http://bitbucket.org/pypy/pypy/changeset/c710232971f1/

Log:	fix tests for -A

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -629,9 +629,15 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "setflags not implemented yet"))
 
-    def descr_sort(self, space, w_axis=-1, w_kind='quicksort', w_order=None):
+    @unwrap_spec(kind=str)
+    def descr_sort(self, space, w_axis=None, kind='quicksort', w_order=None):
+        # happily ignore the kind
+        # modify the array in-place
+        if self.is_scalar():
+            return
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "sort not implemented yet"))
+        return self.implementation.sort(space, w_axis, w_order)
 
     def descr_squeeze(self, space):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -1118,6 +1124,7 @@
     conj = interp2app(W_NDimArray.descr_conj),
 
     argsort  = interp2app(W_NDimArray.descr_argsort),
+    sort  = interp2app(W_NDimArray.descr_sort),
     astype   = interp2app(W_NDimArray.descr_astype),
     base     = GetSetProperty(W_NDimArray.descr_get_base),
     byteswap = interp2app(W_NDimArray.descr_byteswap),
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -68,18 +68,22 @@
         # test doubles and complex doubles as the logic is the same.
 
         # check doubles
-        from numpypy import array, nan, zeros, complex128, arange, empty, dtype
-        msg = "Test real sort order with nans"
+        from numpypy import array, nan, zeros, complex128, arange, dtype
+        from numpy import isnan
         a = array([nan, 1, 0])
-        b = a.sort()
-        assert (b == a[::-1]).all(), msg
+        b = a.copy()
+        b.sort()
+        assert (isnan(b) == isnan(a[::-1])).all()
+        assert (b[:2] == a[::-1][:2]).all()
+
         # check complex
-        msg = "Test complex sort order with nans"
         a = zeros(9, dtype=complex128)
         a.real += [nan, nan, nan, 1, 0, 1, 1, 0, 0]
         a.imag += [nan, 1, 0, nan, nan, 1, 0, 1, 0]
-        b = a.sort()
-        assert (b == a[::-1]).all(), msg
+        b = a.copy()
+        b.sort()
+        assert (isnan(b) == isnan(a[::-1])).all()
+        assert (b[:4] == a[::-1][:4]).all()
 
         # all c scalar sorts use the same code with different types
         # so it suffices to run a quick check with one type. The number
@@ -93,7 +97,6 @@
             c = a.copy();
             c.sort(kind=kind)
             assert (c == a).all(), msg
-            self.assert_equal(c, a, msg)
             c = b.copy();
             c.sort(kind=kind)
             assert (c == a).all(), msg
@@ -229,28 +232,31 @@
             assert (c == a).all(), msg
 
     def test_sort_order(self):
-        from numpypy import array
+        from numpypy import array, zeros
         from sys import byteorder
         # Test sorting an array with fields
         x1 = array([21, 32, 14])
         x2 = array(['my', 'first', 'name'])
         x3=array([3.1, 4.5, 6.2])
-        r=array([x1, x2, x3], dtype=[('id','i'),('word','S'),('number','f')])
+        r=zeros(3, dtype=[('id','i'),('word','S5'),('number','f')])
+        r['id'] = x1
+        r['word'] = x2
+        r['number'] = x3
 
         r.sort(order=['id'])
         assert (r['id'] == [14, 21, 32]).all()
         assert (r['word'] == ['name', 'my', 'first']).all()
-        assert (r['number'] == [6.2, 3.1, 4.5]).all()
+        assert max(abs(r['number'] - [6.2, 3.1, 4.5])) < 1e-6
 
         r.sort(order=['word'])
         assert (r['id'] == [32, 21, 14]).all()
         assert (r['word'] == ['first', 'my', 'name']).all()
-        assert (r['number'] == [4.5, 3.1, 6.2]).all()
+        assert max(abs(r['number'] - [4.5, 3.1, 6.2])) < 1e-6
 
         r.sort(order=['number'])
         assert (r['id'] == [21, 32, 14]).all()
         assert (r['word'] == ['my', 'first', 'name']).all()
-        assert (r['number'] == [3.1, 4.5, 6.2]).all()
+        assert max(abs(r['number'] - [3.1, 4.5, 6.2])) < 1e-6
 
         if byteorder == 'little':
             strtype = '>i2'
@@ -273,11 +279,12 @@
         a = array(range(11),dtype='float64')
         c = a.astype(dtype('<f8'))
         c.sort()
-        assert (a-c).abs().max() < 1e-32
+        assert max(abs(a-c)) < 1e-32
 
     def test_string_sort_with_zeros(self):
         from numpypy import fromstring
         """Check sort for strings containing zeros."""
         x = fromstring("\x00\x02\x00\x01", dtype="S2")
         y = fromstring("\x00\x01\x00\x02", dtype="S2")
-        assert all(x.sort(kind="q") == y)
+        x.sort(kind='q')
+        assert (x == y).all()

From noreply at buildbot.pypy.org  Thu Oct 10 00:07:00 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 00:07:00 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-sort: add a test
Message-ID: <20131009220700.1C1C41C359C@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-sort
Changeset: r67262:3e3e77f4cc44
Date: 2013-10-08 20:05 +0300
http://bitbucket.org/pypy/pypy/changeset/3e3e77f4cc44/

Log:	add a test

diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -60,8 +60,23 @@
         assert (a.argsort(axis=0) == [[1, 0, 0], [0, 1, 1]]).all()
         assert (a.argsort(axis=1) == [[2, 1, 0], [0, 1, 2]]).all()
 
+    def test_sort_dtypes(self):
+        from numpypy import array, arange
+        nnp = self.non_native_prefix
+        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
+                        nnp + 'i2', complex]:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            c = a.copy()
+            a.sort()
+            assert (a == [-1, 3, 3, 4, 6, 8, 100, 101, 256+20]).all(), \
+                'a,orig,dtype %r,%r,%r' % (a,c,dtype)
+            a = arange(100)
+            c = a.copy()
+            assert (a.sort() == c).all()
+
+
 # tests from numpy/tests/test_multiarray.py
-    def test_sort(self):
+    def test_sort_corner_cases(self):
         # test ordering for floats and complex containing nans. It is only
         # necessary to check the lessthan comparison, so sorts that
         # only follow the insertion sort path are sufficient. We only

From noreply at buildbot.pypy.org  Thu Oct 10 00:07:01 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 00:07:01 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-sort: some tests pass,
	raise for non-native byte order
Message-ID: <20131009220701.85EBE1C359C@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-sort
Changeset: r67263:83de01401b30
Date: 2013-10-09 20:24 +0300
http://bitbucket.org/pypy/pypy/changeset/83de01401b30/

Log:	some tests pass, raise for non-native byte order

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -356,6 +356,10 @@
         from pypy.module.micronumpy.arrayimpl.sort import argsort_array
         return argsort_array(self, space, w_axis)
 
+    def sort(self, space, w_axis, w_order):
+        from pypy.module.micronumpy.arrayimpl.sort import sort_array
+        return sort_array(self, space, w_axis, w_order)
+
     def base(self):
         return None
 
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -17,7 +17,7 @@
 
 INT_SIZE = rffi.sizeof(lltype.Signed)
 
-def make_sort_function(space, itemtype, comp_type, count=1):
+def make_argsort_function(space, itemtype, comp_type, count=1):
     TP = itemtype.T
     step = rffi.sizeof(TP)
 
@@ -162,7 +162,7 @@
     return argsort
 
 def argsort_array(arr, space, w_axis):
-    cache = space.fromcache(SortCache) # that populates SortClasses
+    cache = space.fromcache(ArgSortCache) # that populates ArgSortClasses
     itemtype = arr.dtype.itemtype
     for tp in all_types:
         if isinstance(itemtype, tp[0]):
@@ -178,6 +178,161 @@
 all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
 all_types = unrolling_iterable(all_types)
 
+def make_sort_function(space, itemtype, comp_type, count=1):
+    TP = itemtype.T
+    step = rffi.sizeof(TP)
+
+    class Repr(object):
+        def __init__(self, stride_size, size, values, start):
+            self.stride_size = stride_size
+            self.start = start
+            self.size = size
+            self.values = values
+
+        def getitem(self, item):
+            if count < 2:
+                v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                                    + self.start)
+            else:
+                v = []
+                for i in range(count):
+                    _v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                                    + self.start + step * i)
+                    v.append(_v)
+            if comp_type == 'int':
+                v = intmask(v)
+            elif comp_type == 'float':
+                v = float(v)
+            elif comp_type == 'complex':
+                v = [float(v[0]),float(v[1])]
+            else:
+                raise NotImplementedError('cannot reach')
+            return (v)
+
+        def setitem(self, idx, item):
+            if count < 2:
+                raw_storage_setitem(self.values, idx * self.stride_size +
+                                self.start, rffi.cast(TP, item))
+            else:
+                i = 0
+                for val in item:
+                    raw_storage_setitem(self.values, idx * self.stride_size +
+                                self.start + i*step, rffi.cast(TP, val))
+                    i += 1
+
+    class ArgArrayRepWithStorage(Repr):
+        def __init__(self, stride_size, size):
+            start = 0
+            values = alloc_raw_storage(size * stride_size,
+                                            track_allocation=False)
+            Repr.__init__(self, stride_size,
+                          size, values, start)
+
+        def __del__(self):
+            free_raw_storage(self.values, track_allocation=False)
+
+    def arg_getitem(lst, item):
+        return lst.getitem(item)
+
+    def arg_setitem(lst, item, value):
+        lst.setitem(item, value)
+
+    def arg_length(lst):
+        return lst.size
+
+    def arg_getitem_slice(lst, start, stop):
+        retval = ArgArrayRepWithStorage(lst.stride_size, stop-start)
+        for i in range(stop-start):
+            retval.setitem(i, lst.getitem(i+start))
+        return retval
+
+    if count < 2:
+        def arg_lt(a, b):
+            # handles NAN and INF
+            return a < b or b != b and a == a
+    else:
+        def arg_lt(a, b):
+            for i in range(count):
+                if a[i] < b[i] or b != b and a == a:
+                    return True
+                elif a[i] > b[i]:
+                    return False
+            # Does numpy do True?
+            return False
+
+    ArgSort = make_timsort_class(arg_getitem, arg_setitem, arg_length,
+                                 arg_getitem_slice, arg_lt)
+
+    def sort(arr, space, w_axis, itemsize):
+        if w_axis is space.w_None:
+            # note that it's fine ot pass None here as we're not going
+            # to pass the result around (None is the link to base in slices)
+            arr = arr.reshape(space, None, [arr.get_size()])
+            axis = 0
+        elif w_axis is None:
+            axis = -1
+        else:
+            axis = space.int_w(w_axis)
+        # create array of indexes
+        if len(arr.get_shape()) == 1:
+            r = Repr(itemsize, arr.get_size(), arr.get_storage(),
+                     arr.start)
+            ArgSort(r).sort()
+        else:
+            shape = arr.get_shape()
+            if axis < 0:
+                axis = len(shape) + axis - 1
+            if axis < 0 or axis > len(shape):
+                raise OperationError(space.w_IndexError, space.wrap(
+                                                    "Wrong axis %d" % axis))
+            iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
+            iter = AxisIterator(arr, iterable_shape, axis, False)
+            stride_size = arr.strides[axis]
+            axis_size = arr.shape[axis]
+            while not iter.done():
+                r = Repr(stride_size, axis_size, arr.get_storage(), iter.offset)
+                ArgSort(r).sort()
+                iter.next()
+
+    return sort
+
+def sort_array(arr, space, w_axis, w_order):
+    cache = space.fromcache(SortCache) # that populates SortClasses
+    itemtype = arr.dtype.itemtype
+    if not arr.dtype.native:
+        raise OperationError(space.w_NotImplementedError,
+           space.wrap("sorting of non-native btyeorder not supported yet"))
+    for tp in all_types:
+        if isinstance(itemtype, tp[0]):
+            return cache._lookup(tp)(arr, space, w_axis,
+                                     itemtype.get_element_size())
+    # XXX this should probably be changed
+    raise OperationError(space.w_NotImplementedError,
+           space.wrap("sorting of non-numeric types " + \
+                  "'%s' is not implemented" % arr.dtype.get_name(), ))
+
+all_types = (types.all_float_types + types.all_complex_types +
+             types.all_int_types)
+all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
+all_types = unrolling_iterable(all_types)
+
+class ArgSortCache(object):
+    built = False
+
+    def __init__(self, space):
+        if self.built:
+            return
+        self.built = True
+        cache = {}
+        for cls, it in all_types._items:
+            if it == 'complex':
+                cache[cls] = make_argsort_function(space, cls, it, 2)
+            else:
+                cache[cls] = make_argsort_function(space, cls, it)
+        self.cache = cache
+        self._lookup = specialize.memo()(lambda tp : cache[tp[0]])
+
+
 class SortCache(object):
     built = False
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -635,8 +635,6 @@
         # modify the array in-place
         if self.is_scalar():
             return
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "sort not implemented yet"))
         return self.implementation.sort(space, w_axis, w_order)
 
     def descr_squeeze(self, space):
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -62,17 +62,30 @@
 
     def test_sort_dtypes(self):
         from numpypy import array, arange
-        nnp = self.non_native_prefix
         for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
-                        nnp + 'i2', complex]:
+                        'i2', complex]:
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
             c = a.copy()
             a.sort()
-            assert (a == [-1, 3, 3, 4, 6, 8, 100, 101, 256+20]).all(), \
+            assert (a == b).all(), \
                 'a,orig,dtype %r,%r,%r' % (a,c,dtype)
-            a = arange(100)
+        a = arange(100)
+        c = a.copy()
+        a.sort()
+        assert (a == c).all()
+
+    def test_sort_dtypesi_nonnative(self):
+        from numpypy import array
+        nnp = self.non_native_prefix
+        for dtype in [ nnp + 'i2']:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
             c = a.copy()
-            assert (a.sort() == c).all()
+            exc = raises(NotImplementedError, a.sort)
+            assert exc.value[0].find('supported') >= 0
+            #assert (a == b).all(), \
+            #    'a,orig,dtype %r,%r,%r' % (a,c,dtype)
 
 
 # tests from numpy/tests/test_multiarray.py
@@ -286,8 +299,6 @@
                                  dtype=mydtype)).all()
 
 
-
-
 # tests from numpy/tests/test_regression.py
     def test_sort_bigendian(self):
         from numpypy import array, dtype

From noreply at buildbot.pypy.org  Thu Oct 10 00:07:02 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 00:07:02 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-sort: fix sorting nan for complex
Message-ID: <20131009220702.B9E931C359C@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-sort
Changeset: r67264:bde7f67cd9db
Date: 2013-10-09 22:58 +0300
http://bitbucket.org/pypy/pypy/changeset/bde7f67cd9db/

Log:	fix sorting nan for complex

diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -253,7 +253,12 @@
     else:
         def arg_lt(a, b):
             for i in range(count):
-                if a[i] < b[i] or b != b and a == a:
+                if b[i] != b[i] and a[i] == a[i]:
+                    return True
+                elif b[i] == b[i] and a[i] != a[i]:
+                    return False
+            for i in range(count):
+                if a[i] < b[i]:
                     return True
                 elif a[i] > b[i]:
                     return False

From noreply at buildbot.pypy.org  Thu Oct 10 00:07:03 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 00:07:03 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-sort: fix default axis off-by-one
Message-ID: <20131009220703.EA7D61C359C@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-sort
Changeset: r67265:656eded4a6ba
Date: 2013-10-09 23:13 +0300
http://bitbucket.org/pypy/pypy/changeset/656eded4a6ba/

Log:	fix default axis off-by-one

diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -137,8 +137,8 @@
         else:
             shape = arr.get_shape()
             if axis < 0:
-                axis = len(shape) + axis - 1
-            if axis < 0 or axis > len(shape):
+                axis = len(shape) + axis
+            if axis < 0 or axis >= len(shape):
                 raise OperationError(space.w_IndexError, space.wrap(
                                                     "Wrong axis %d" % axis))
             iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
@@ -270,7 +270,7 @@
 
     def sort(arr, space, w_axis, itemsize):
         if w_axis is space.w_None:
-            # note that it's fine ot pass None here as we're not going
+            # note that it's fine to pass None here as we're not going
             # to pass the result around (None is the link to base in slices)
             arr = arr.reshape(space, None, [arr.get_size()])
             axis = 0
@@ -286,8 +286,8 @@
         else:
             shape = arr.get_shape()
             if axis < 0:
-                axis = len(shape) + axis - 1
-            if axis < 0 or axis > len(shape):
+                axis = len(shape) + axis
+            if axis < 0 or axis >= len(shape):
                 raise OperationError(space.w_IndexError, space.wrap(
                                                     "Wrong axis %d" % axis))
             iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -96,7 +96,7 @@
         # test doubles and complex doubles as the logic is the same.
 
         # check doubles
-        from numpypy import array, nan, zeros, complex128, arange, dtype
+        from numpypy import array, nan, zeros, complex128, arange
         from numpy import isnan
         a = array([nan, 1, 0])
         b = a.copy()
@@ -152,19 +152,6 @@
             c.sort(kind=kind)
             assert (c == ai).all(), msg
 
-        # test string sorts.
-        s = 'aaaaaaaa'
-        a = array([s + chr(i) for i in range(101)])
-        b = a[::-1].copy()
-        for kind in ['q', 'm', 'h'] :
-            msg = "string sort, kind=%s" % kind
-            c = a.copy();
-            c.sort(kind=kind)
-            assert (c == a).all(), msg
-            c = b.copy();
-            c.sort(kind=kind)
-            assert (c == a).all(), msg
-
         # check axis handling. This should be the same for all type
         # specific sorts, so we only check it for one type and one kind
         a = array([[3, 2], [1, 0]])
@@ -180,6 +167,22 @@
         d.sort()
         assert (d == c).all(), "test sort with default axis"
 
+    def test_sort_corner_cases_string_records(self):
+        skip('not implemented yet')
+        from numpypy import array, dtype
+        # test string sorts.
+        s = 'aaaaaaaa'
+        a = array([s + chr(i) for i in range(101)])
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h'] :
+            msg = "string sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
 
         # test record array sorts.
         dt =dtype([('f', float), ('i', int)])
@@ -263,6 +266,7 @@
         from numpypy import array, zeros
         from sys import byteorder
         # Test sorting an array with fields
+        skip('not implemented yet')
         x1 = array([21, 32, 14])
         x2 = array(['my', 'first', 'name'])
         x3=array([3.1, 4.5, 6.2])
@@ -301,6 +305,7 @@
 
 # tests from numpy/tests/test_regression.py
     def test_sort_bigendian(self):
+        skip('not implemented yet')
         from numpypy import array, dtype
         a = array(range(11),dtype='float64')
         c = a.astype(dtype('<f8'))
@@ -308,6 +313,7 @@
         assert max(abs(a-c)) < 1e-32
 
     def test_string_sort_with_zeros(self):
+        skip('not implemented yet')
         from numpypy import fromstring
         """Check sort for strings containing zeros."""
         x = fromstring("\x00\x02\x00\x01", dtype="S2")

From noreply at buildbot.pypy.org  Thu Oct 10 00:07:05 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 00:07:05 +0200 (CEST)
Subject: [pypy-commit] pypy cpyext-best_base: add test,
	fix best_base (amaury)
Message-ID: <20131009220705.2C0701C359C@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-best_base
Changeset: r67266:00e1a7288bbb
Date: 2013-10-10 01:03 +0300
http://bitbucket.org/pypy/pypy/changeset/00e1a7288bbb/

Log:	add test, fix best_base (amaury)

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -99,7 +99,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii'])
+                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
@@ -196,7 +196,7 @@
         assert PyUnicode_GetDefaultEncoding() == 'ascii'
 
 class AppTestCpythonExtensionBase(LeakCheckingTest):
-    
+
     def setup_class(cls):
         cls.space.getbuiltinmodule("cpyext")
         from pypy.module.imp.importing import importhook
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -156,7 +156,7 @@
             def __init__(self):
                 self.foobar = 32
                 super(UnicodeSubclass2, self).__init__()
-        
+
         newobj = UnicodeSubclass2()
         assert newobj.get_val() == 42
         assert newobj.foobar == 32
@@ -358,6 +358,13 @@
         assert w_obj is None
         assert api.PyErr_Occurred() is None
 
+    def test_ndarray_ref(self, space, api):
+        w_obj = space.appexec([], """():
+            import numpypy as np
+            return np.int64(2)""")
+        ref = make_ref(space, w_obj)
+        api.Py_DecRef(ref)
+
 class AppTestSlots(AppTestCpythonExtensionBase):
     def test_some_slots(self):
         module = self.import_extension('foo', [
@@ -525,7 +532,7 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
-        
+
     def test_bool(self):
         module = self.import_extension('foo', [
             ("newInt", "METH_VARARGS",
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -4,7 +4,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.annlowlevel import llhelper
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
-from pypy.objspace.std.typeobject import W_TypeObject
+from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
@@ -575,6 +575,7 @@
     if not bases_w:
         return None
 
+    return find_best_base(space, bases_w)
     w_winner = None
     w_base = None
     for w_base_i in bases_w:

From noreply at buildbot.pypy.org  Thu Oct 10 00:07:06 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 00:07:06 +0200 (CEST)
Subject: [pypy-commit] pypy cpyext-best_base: cleanup
Message-ID: <20131009220706.4ABA01C359C@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-best_base
Changeset: r67267:e409b611cf6d
Date: 2013-10-10 01:05 +0300
http://bitbucket.org/pypy/pypy/changeset/e409b611cf6d/

Log:	cleanup

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -574,34 +574,7 @@
 def best_base(space, bases_w):
     if not bases_w:
         return None
-
     return find_best_base(space, bases_w)
-    w_winner = None
-    w_base = None
-    for w_base_i in bases_w:
-        if isinstance(w_base_i, W_ClassObject):
-            # old-style base
-            continue
-        assert isinstance(w_base_i, W_TypeObject)
-        w_candidate = solid_base(space, w_base_i)
-        if not w_winner:
-            w_winner = w_candidate
-            w_base = w_base_i
-        elif space.abstract_issubclass_w(w_winner, w_candidate):
-            pass
-        elif space.abstract_issubclass_w(w_candidate, w_winner):
-            w_winner = w_candidate
-            w_base = w_base_i
-        else:
-            raise OperationError(
-                space.w_TypeError,
-                space.wrap("multiple bases have instance lay-out conflict"))
-    if w_base is None:
-        raise OperationError(
-            space.w_TypeError,
-                space.wrap("a new-style class can't have only classic bases"))
-
-    return w_base
 
 def inherit_slots(space, pto, w_base):
     # XXX missing: nearly everything

From noreply at buildbot.pypy.org  Thu Oct 10 00:45:55 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 00:45:55 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-sort: merge default into branch
Message-ID: <20131009224555.7606D1C33FE@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-sort
Changeset: r67268:3315d5cfdbca
Date: 2013-10-10 01:09 +0300
http://bitbucket.org/pypy/pypy/changeset/3315d5cfdbca/

Log:	merge default into branch

diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -1,1 +1,75 @@
 from _numpypy.numerictypes import *
+import numpypy
+
+def issubclass_(arg1, arg2):
+    """
+    Determine if a class is a subclass of a second class.
+
+    `issubclass_` is equivalent to the Python built-in ``issubclass``,
+    except that it returns False instead of raising a TypeError is one
+    of the arguments is not a class.
+
+    Parameters
+    ----------
+    arg1 : class
+        Input class. True is returned if `arg1` is a subclass of `arg2`.
+    arg2 : class or tuple of classes.
+        Input class. If a tuple of classes, True is returned if `arg1` is a
+        subclass of any of the tuple elements.
+
+    Returns
+    -------
+    out : bool
+        Whether `arg1` is a subclass of `arg2` or not.
+
+    See Also
+    --------
+    issubsctype, issubdtype, issctype
+
+    Examples
+    --------
+    >>> np.issubclass_(np.int32, np.int)
+    True
+    >>> np.issubclass_(np.int32, np.float)
+    False
+
+    """
+    try:
+        return issubclass(arg1, arg2)
+    except TypeError:
+        return False
+
+def issubdtype(arg1, arg2):
+    """
+    Returns True if first argument is a typecode lower/equal in type hierarchy.
+
+    Parameters
+    ----------
+    arg1, arg2 : dtype_like
+        dtype or string representing a typecode.
+
+    Returns
+    -------
+    out : bool
+
+    See Also
+    --------
+    issubsctype, issubclass_
+    numpy.core.numerictypes : Overview of numpy type hierarchy.
+
+    Examples
+    --------
+    >>> np.issubdtype('S1', str)
+    True
+    >>> np.issubdtype(np.float64, np.float32)
+    False
+
+    """
+    if issubclass_(arg2, generic):
+        return issubclass(numpypy.dtype(arg1).type, arg2)
+    mro = numpypy.dtype(arg2).type.mro()
+    if len(mro) > 1:
+        val = mro[1]
+    else:
+        val = mro[0]
+    return issubclass(numpypy.dtype(arg1).type, val)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -400,6 +400,8 @@
     def test_socket_connect_ex(self):
         import _socket
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+        # The following might fail if the DNS redirects failed requests to a
+        # catch-all address (i.e. opendns).
         # Make sure we get an app-level error, not an interp one.
         raises(_socket.gaierror, s.connect_ex, ("wrong.invalid", 80))
         s.close()
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -103,11 +103,13 @@
     exc_p[0] = make_ref(space, operr.w_type)
     val_p[0] = make_ref(space, operr.get_w_value(space))
 
- at cpython_api([], lltype.Void)
+ at cpython_api([], rffi.INT_real, error=0)
 def PyErr_BadArgument(space):
     """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where
     message indicates that a built-in operation was invoked with an illegal
-    argument.  It is mostly for internal use."""
+    argument.  It is mostly for internal use. In CPython this function always
+    raises an exception and returns 0 in all cases, hence the (ab)use of the
+    error indicator."""
     raise OperationError(space.w_TypeError,
             space.wrap("bad argument type for built-in operation"))
 
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -70,9 +70,10 @@
         api.PyErr_Clear()
 
     def test_BadArgument(self, space, api):
-        api.PyErr_BadArgument()
+        ret = api.PyErr_BadArgument()
         state = space.fromcache(State)
         assert space.eq_w(state.operror.w_type, space.w_TypeError)
+        assert ret == 0
         api.PyErr_Clear()
 
     def test_Warning(self, space, api, capfd):
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -91,6 +91,7 @@
         invalid = rffi.str2charp('invalid')
         utf_8 = rffi.str2charp('utf-8')
         prev_encoding = rffi.str2charp(space.unwrap(w_default_encoding))
+        self.raises(space, api, TypeError, api.PyUnicode_SetDefaultEncoding, lltype.nullptr(rffi.CCHARP.TO))
         assert api.PyUnicode_SetDefaultEncoding(invalid) == -1
         assert api.PyErr_Occurred() is space.w_LookupError
         api.PyErr_Clear()
@@ -316,6 +317,15 @@
         rffi.free_charp(b_text)
         rffi.free_charp(b_encoding)
 
+    def test_decode_null_encoding(self, space, api):
+        null_charp = lltype.nullptr(rffi.CCHARP.TO)
+        u_text = u'abcdefg'
+        s_text = space.str_w(api.PyUnicode_AsEncodedString(space.wrap(u_text), null_charp, null_charp))
+        b_text = rffi.str2charp(s_text)
+        assert space.unwrap(api.PyUnicode_Decode(b_text, len(s_text), null_charp, null_charp)) == u_text
+        self.raises(space, api, TypeError, api.PyUnicode_FromEncodedObject, space.wrap(u_text), null_charp, None)
+        rffi.free_charp(b_text)
+
     def test_leak(self):
         size = 50
         raw_buf, gc_buf = rffi.alloc_buffer(size)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -273,6 +273,8 @@
 def PyUnicode_SetDefaultEncoding(space, encoding):
     """Sets the currently active default encoding. Returns 0 on
     success, -1 in case of an error."""
+    if not encoding:
+        PyErr_BadArgument(space)
     w_encoding = space.wrap(rffi.charp2str(encoding))
     setdefaultencoding(space, w_encoding)
     default_encoding[0] = '\x00'
@@ -350,8 +352,11 @@
     in the unicode() built-in function.  The codec to be used is looked up
     using the Python codec registry.  Return NULL if an exception was raised by
     the codec."""
+    if not encoding:
+        # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead
+        encoding = PyUnicode_GetDefaultEncoding(space)
+    w_encoding = space.wrap(rffi.charp2str(encoding))
     w_str = space.wrap(rffi.charpsize2str(s, size))
-    w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))
     else:
@@ -379,6 +384,9 @@
 
     All other objects, including Unicode objects, cause a TypeError to be
     set."""
+    if not encoding:
+        raise OperationError(space.w_TypeError,
+                             space.wrap("decoding Unicode is not supported"))
     w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -319,6 +319,15 @@
         else:
             self.done_func = None
 
+    def are_common_types(self, dtype1, dtype2):
+        if dtype1.is_complex_type() and dtype2.is_complex_type():
+            return True
+        elif not (dtype1.is_complex_type() or dtype2.is_complex_type()) and \
+                (dtype1.is_int_type() and dtype2.is_int_type() or dtype1.is_float_type() and dtype2.is_float_type()) and \
+                not (dtype1.is_bool_type() or dtype2.is_bool_type()):
+            return True
+        return False
+
     @jit.unroll_safe
     def call(self, space, args_w):
         if len(args_w) > 2:
@@ -339,6 +348,12 @@
                  'unsupported operand dtypes %s and %s for "%s"' % \
                  (w_rdtype.get_name(), w_ldtype.get_name(),
                   self.name)))
+
+        if self.are_common_types(w_ldtype, w_rdtype):
+            if not w_lhs.is_scalar() and w_rhs.is_scalar():
+                w_rdtype = w_ldtype
+            elif w_lhs.is_scalar() and not w_rhs.is_scalar():
+                w_ldtype = w_rdtype
         calc_dtype = find_binop_result_dtype(space,
             w_ldtype, w_rdtype,
             int_only=self.int_only,
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2922,6 +2922,11 @@
                         dtype=[('bg', 'i8'), ('fg', 'i8'), ('char', 'S1')])
         assert c[0][0]["char"] == 'a'
 
+    def test_scalar_coercion(self):
+        import numpypy as np
+        a = np.array([1,2,3], dtype=np.int16)
+        assert (a * 2).dtype == np.int16
+
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -149,6 +149,8 @@
         interpleveldefs['nice'] = 'interp_posix.nice'
     if hasattr(os, 'getlogin'):
         interpleveldefs['getlogin'] = 'interp_posix.getlogin'
+    if hasattr(os, 'ctermid'):
+        interpleveldefs['ctermid'] = 'interp_posix.ctermid'
 
     for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid',
                  'seteuid', 'setgid', 'setegid', 'getgroups', 'getpgrp',
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1218,3 +1218,10 @@
         return space.wrap(rurandom.urandom(context, n))
     except OSError, e:
         raise wrap_oserror(space, e)
+
+def ctermid(space):
+    """ctermid() -> string
+
+    Return the name of the controlling terminal for this process.
+    """
+    return space.wrap(os.ctermid())
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -110,7 +110,7 @@
                        '__pypy__', 'cStringIO', '_collections', 'struct',
                        'mmap', 'marshal', '_codecs', 'rctime', 'cppyy',
                        '_cffi_backend', 'pyexpat', '_continuation', '_io',
-                       'thread']:
+                       'thread', 'select']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -49,12 +49,15 @@
     from pypy.module.thread.os_lock import Lock
     assert pypypolicy.look_inside_function(Lock.descr_lock_acquire.im_func)
 
+def test_select():
+    from pypy.module.select.interp_select import poll
+    assert pypypolicy.look_inside_function(poll)
+
 def test_pypy_module():
     from pypy.module._collections.interp_deque import W_Deque
     from pypy.module._random.interp_random import W_Random
     assert not pypypolicy.look_inside_function(W_Random.random)
     assert pypypolicy.look_inside_function(W_Deque.length)
-    assert not pypypolicy.look_inside_pypy_module('select.interp_epoll')
     assert pypypolicy.look_inside_pypy_module('__builtin__.operation')
     assert pypypolicy.look_inside_pypy_module('__builtin__.abstractinst')
     assert pypypolicy.look_inside_pypy_module('__builtin__.functional')
diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -180,12 +180,12 @@
                         i = 0
                         for w_ev in space.listview(w_changelist):
                             ev = space.interp_w(W_Kevent, w_ev)
-                            changelist[i].c_ident = ev.event.c_ident
-                            changelist[i].c_filter = ev.event.c_filter
-                            changelist[i].c_flags = ev.event.c_flags
-                            changelist[i].c_fflags = ev.event.c_fflags
-                            changelist[i].c_data = ev.event.c_data
-                            changelist[i].c_udata = ev.event.c_udata
+                            changelist[i].c_ident = ev.ident
+                            changelist[i].c_filter = ev.filter
+                            changelist[i].c_flags = ev.flags
+                            changelist[i].c_fflags = ev.fflags
+                            changelist[i].c_data = ev.data
+                            changelist[i].c_udata = ev.udata
                             i += 1
                         pchangelist = changelist
                     else:
@@ -206,13 +206,12 @@
                             evt = eventlist[i]
 
                             w_event = W_Kevent(space)
-                            w_event.event = lltype.malloc(kevent, flavor="raw")
-                            w_event.event.c_ident = evt.c_ident
-                            w_event.event.c_filter = evt.c_filter
-                            w_event.event.c_flags = evt.c_flags
-                            w_event.event.c_fflags = evt.c_fflags
-                            w_event.event.c_data = evt.c_data
-                            w_event.event.c_udata = evt.c_udata
+                            w_event.ident = evt.c_ident
+                            w_event.filter = evt.c_filter
+                            w_event.flags = evt.c_flags
+                            w_event.fflags = evt.c_fflags
+                            w_event.data = evt.c_data
+                            w_event.udata = evt.c_udata
 
                             elist_w[i] = w_event
 
@@ -234,11 +233,12 @@
 
 class W_Kevent(W_Root):
     def __init__(self, space):
-        self.event = lltype.nullptr(kevent)
-
-    def __del__(self):
-        if self.event:
-            lltype.free(self.event, flavor="raw")
+        self.ident = rffi.cast(kevent.c_ident, 0)
+        self.filter = rffi.cast(kevent.c_filter, 0)
+        self.flags = rffi.cast(kevent.c_flags, 0)
+        self.fflags = rffi.cast(kevent.c_fflags, 0)
+        self.data = rffi.cast(kevent.c_data, 0)
+        self.udata = lltype.nullptr(rffi.VOIDP.TO)
 
     @unwrap_spec(filter=int, flags='c_uint', fflags='c_uint', data=int, udata=r_uint)
     def descr__init__(self, space, w_ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=r_uint(0)):
@@ -247,35 +247,34 @@
         else:
             ident = r_uint(space.c_filedescriptor_w(w_ident))
 
-        self.event = lltype.malloc(kevent, flavor="raw")
-        rffi.setintfield(self.event, "c_ident", ident)
-        rffi.setintfield(self.event, "c_filter", filter)
-        rffi.setintfield(self.event, "c_flags", flags)
-        rffi.setintfield(self.event, "c_fflags", fflags)
-        rffi.setintfield(self.event, "c_data", data)
-        self.event.c_udata = rffi.cast(rffi.VOIDP, udata)
+        self.ident = rffi.cast(kevent.c_ident, ident)
+        self.filter = rffi.cast(kevent.c_filter, filter)
+        self.flags = rffi.cast(kevent.c_flags, flags)
+        self.fflags = rffi.cast(kevent.c_fflags, fflags)
+        self.data = rffi.cast(kevent.c_data, data)
+        self.udata = rffi.cast(rffi.VOIDP, udata)
 
     def _compare_all_fields(self, other, op):
         if IDENT_UINT:
-            l_ident = rffi.cast(lltype.Unsigned, self.event.c_ident)
-            r_ident = rffi.cast(lltype.Unsigned, other.event.c_ident)
+            l_ident = rffi.cast(lltype.Unsigned, self.ident)
+            r_ident = rffi.cast(lltype.Unsigned, other.ident)
         else:
-            l_ident = self.event.c_ident
-            r_ident = other.event.c_ident
-        l_filter = rffi.cast(lltype.Signed, self.event.c_filter)
-        r_filter = rffi.cast(lltype.Signed, other.event.c_filter)
-        l_flags = rffi.cast(lltype.Unsigned, self.event.c_flags)
-        r_flags = rffi.cast(lltype.Unsigned, other.event.c_flags)
-        l_fflags = rffi.cast(lltype.Unsigned, self.event.c_fflags)
-        r_fflags = rffi.cast(lltype.Unsigned, other.event.c_fflags)
+            l_ident = self.ident
+            r_ident = other.ident
+        l_filter = rffi.cast(lltype.Signed, self.filter)
+        r_filter = rffi.cast(lltype.Signed, other.filter)
+        l_flags = rffi.cast(lltype.Unsigned, self.flags)
+        r_flags = rffi.cast(lltype.Unsigned, other.flags)
+        l_fflags = rffi.cast(lltype.Unsigned, self.fflags)
+        r_fflags = rffi.cast(lltype.Unsigned, other.fflags)
         if IDENT_UINT:
-            l_data = rffi.cast(lltype.Signed, self.event.c_data)
-            r_data = rffi.cast(lltype.Signed, other.event.c_data)
+            l_data = rffi.cast(lltype.Signed, self.data)
+            r_data = rffi.cast(lltype.Signed, other.data)
         else:
-            l_data = self.event.c_data
-            r_data = other.event.c_data
-        l_udata = rffi.cast(lltype.Unsigned, self.event.c_udata)
-        r_udata = rffi.cast(lltype.Unsigned, other.event.c_udata)
+            l_data = self.data
+            r_data = other.data
+        l_udata = rffi.cast(lltype.Unsigned, self.udata)
+        r_udata = rffi.cast(lltype.Unsigned, other.udata)
 
         if op == "eq":
             return l_ident == r_ident and \
@@ -330,22 +329,22 @@
         return space.wrap(self.compare_all_fields(space, w_other, "gt"))
 
     def descr_get_ident(self, space):
-        return space.wrap(self.event.c_ident)
+        return space.wrap(self.ident)
 
     def descr_get_filter(self, space):
-        return space.wrap(self.event.c_filter)
+        return space.wrap(self.filter)
 
     def descr_get_flags(self, space):
-        return space.wrap(self.event.c_flags)
+        return space.wrap(self.flags)
 
     def descr_get_fflags(self, space):
-        return space.wrap(self.event.c_fflags)
+        return space.wrap(self.fflags)
 
     def descr_get_data(self, space):
-        return space.wrap(self.event.c_data)
+        return space.wrap(self.data)
 
     def descr_get_udata(self, space):
-        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.event.c_udata))
+        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.udata))
 
 
 W_Kevent.typedef = TypeDef("select.kevent",
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -30,7 +30,7 @@
 from rpython.rlib import debug, jit, rerased
 from rpython.rlib.listsort import make_timsort_class
 from rpython.rlib.objectmodel import (
-    instantiate, newlist_hint, resizelist_hint, specialize)
+    instantiate, newlist_hint, resizelist_hint, specialize, import_from_mixin)
 from rpython.tool.sourcetools import func_with_new_name
 
 __all__ = ['W_ListObject', 'make_range_list', 'make_empty_list_with_size']
@@ -1170,7 +1170,6 @@
 
 
 class AbstractUnwrappedStrategy(object):
-    _mixin_ = True
 
     def wrap(self, unwrapped):
         raise NotImplementedError
@@ -1329,7 +1328,6 @@
 
     def setslice(self, w_list, start, step, slicelength, w_other):
         assert slicelength >= 0
-        items = self.unerase(w_list.lstorage)
 
         if self is self.space.fromcache(ObjectListStrategy):
             w_other = w_other._temporarily_as_objects()
@@ -1341,6 +1339,7 @@
             w_list.setslice(start, step, slicelength, w_other_as_object)
             return
 
+        items = self.unerase(w_list.lstorage)
         oldsize = len(items)
         len2 = w_other.length()
         if step == 1:  # Support list resizing for non-extended slices
@@ -1456,7 +1455,9 @@
         self.unerase(w_list.lstorage).reverse()
 
 
-class ObjectListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class ObjectListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "object"
 
@@ -1489,7 +1490,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class IntegerListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class IntegerListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0
     _applevel_repr = "int"
 
@@ -1520,7 +1523,30 @@
         return self.unerase(w_list.lstorage)
 
 
-class FloatListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            l = self.unerase(w_list.lstorage)
+            other = w_other.getitems_int()
+            assert other is not None
+            l += other
+            return
+        return self._base_extend_from_list(w_list, w_other)
+
+
+    _base_setslice = setslice
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            storage = self.erase(w_other.getitems_int())
+            w_other = W_ListObject.from_storage_and_strategy(
+                    self.space, storage, self)
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
+class FloatListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0.0
     _applevel_repr = "float"
 
@@ -1548,7 +1574,9 @@
             l.reverse()
 
 
-class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class StringListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "str"
 
@@ -1579,7 +1607,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class UnicodeListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class UnicodeListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "unicode"
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -225,6 +225,15 @@
         l.setslice(0, 1, 2, W_ListObject(space, [w('a'), w(2), w(3)]))
         assert isinstance(l.strategy, ObjectListStrategy)
 
+    def test_setslice_int_range(self):
+        space = self.space
+        w = space.wrap
+        l = W_ListObject(space, [w(1), w(2), w(3)])
+        assert isinstance(l.strategy, IntegerListStrategy)
+        l.setslice(0, 1, 2, make_range_list(space, 5, 1, 4))
+        assert isinstance(l.strategy, IntegerListStrategy)
+
+
     def test_setslice_List(self):
         space = self.space
 
@@ -467,6 +476,12 @@
         l4 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3), self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         assert self.space.eq_w(l3, l4)
 
+    def test_add_of_range_and_int(self):
+        l1 = make_range_list(self.space, 0, 1, 100)
+        l2 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
+        l3 = self.space.add(l2, l1)
+        assert l3.strategy is l2.strategy
+
     def test_mul(self):
         l1 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         l2 = l1.mul(2)
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3442,6 +3442,29 @@
 
         a.build_types(f, [str])
 
+    def test_negative_number_find(self):
+        def f(s, e):
+            return "xyz".find("x", s, e)
+
+        a = self.RPythonAnnotator()
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        def f(s, e):
+            return "xyz".rfind("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+
+        def f(s, e):
+            return "xyz".count("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        
+
     def test_setslice(self):
         def f():
             lst = [2, 5, 7]
@@ -4080,7 +4103,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify instances with no common base class" 
+        assert ("RPython cannot unify instances with no common base class"
                 in exc.value.msg)
 
     def test_unionerror_iters(self):
@@ -4096,7 +4119,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify incompatible iterator variants" in 
+        assert ("RPython cannot unify incompatible iterator variants" in
                 exc.value.msg)
 
     def test_variable_getattr(self):
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -10,7 +10,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeFloat, SomeIterator,
     SomePBC, SomeTypedAddressAccess, SomeAddress, SomeType, s_ImpossibleValue,
     s_Bool, s_None, unionof, missing_operation, add_knowntypedata,
-    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString)
+    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper
 from rpython.annotator import builtin
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
@@ -333,12 +333,13 @@
         check_negative_slice(s_start, s_stop)
         lst.listdef.resize()
 
-def check_negative_slice(s_start, s_stop):
+def check_negative_slice(s_start, s_stop, error="slicing"):
     if isinstance(s_start, SomeInteger) and not s_start.nonneg:
-        raise AnnotatorError("slicing: not proven to have non-negative start")
+        raise AnnotatorError("%s: not proven to have non-negative start" %
+                             error)
     if isinstance(s_stop, SomeInteger) and not s_stop.nonneg and \
            getattr(s_stop, 'const', 0) != -1:
-        raise AnnotatorError("slicing: not proven to have non-negative stop")
+        raise AnnotatorError("%s: not proven to have non-negative stop" % error)
 
 
 class __extend__(SomeDict):
@@ -448,12 +449,15 @@
         return s_Bool
 
     def method_find(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "find")
         return SomeInteger()
 
     def method_rfind(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "rfind")
         return SomeInteger()
 
     def method_count(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "count")
         return SomeInteger(nonneg=True)
 
     def method_strip(str, chr):
@@ -520,6 +524,11 @@
     op_contains.can_only_throw = []
 
 
+class __extend__(SomeByteArray):
+    def getslice(ba, s_start, s_stop):
+        check_negative_slice(s_start, s_stop)
+        return SomeByteArray()
+
 class __extend__(SomeUnicodeString):
     def method_encode(uni, s_enc):
         if not s_enc.is_constant():
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -252,6 +252,23 @@
         from rpython.translator.tool.graphpage import try_show
         try_show(self)
 
+    def get_graph(self):
+        import gc
+        pending = [self]   # pending blocks
+        seen = {self: True, None: True}
+        for x in pending:
+            for y in gc.get_referrers(x):
+                if isinstance(y, FunctionGraph):
+                    return y
+                elif isinstance(y, Link):
+                    block = y.prevblock
+                    if block not in seen:
+                        pending.append(block)
+                        seen[block] = True
+                elif isinstance(y, dict):
+                    pending.append(y)   # go back from the dict to the real obj
+        return pending
+
     view = show
 
 
diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py
--- a/rpython/flowspace/specialcase.py
+++ b/rpython/flowspace/specialcase.py
@@ -1,3 +1,4 @@
+import os
 from rpython.flowspace.model import Constant, const
 
 SPECIAL_CASES = {}
@@ -37,6 +38,18 @@
     return space.frame.do_operation('simple_call', const(isinstance),
             w_instance, w_type)
 
+ at register_flow_sc(open)
+def sc_open(space, *args_w):
+    from rpython.rlib.rfile import create_file
+
+    return space.frame.do_operation("simple_call", const(create_file), *args_w)
+
+ at register_flow_sc(os.tmpfile)
+def sc_os_tmpfile(space):
+    from rpython.rlib.rfile import create_temp_rfile
+
+    return space.frame.do_operation("simple_call", const(create_temp_rfile))
+
 # _________________________________________________________________________
 # a simplified version of the basic printing routines, for RPython programs
 class StdOutBuffer:
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1201,6 +1201,7 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+            assert self.card_page_indices > 0
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -1,55 +1,241 @@
 
-""" This file makes open() and friends RPython
+""" This file makes open() and friends RPython. Note that RFile should not
+be used directly and instead it's magically appearing each time you call
+python builtin open()
 """
 
 import os
-from rpython.annotator.model import SomeObject, SomeString, SomeInteger
-from rpython.rtyper.extregistry import ExtRegistryEntry
-from rpython.rtyper.extfunc import register_external
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.tool import rffi_platform as platform
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib import rposix
+from rpython.rlib.rstring import StringBuilder
 
-class SomeFile(SomeObject):
-    def method_write(self, s_arg):
-        assert isinstance(s_arg, SomeString)
+eci = ExternalCompilationInfo(includes=['stdio.h', 'unistd.h', 'sys/types.h'])
 
-    def method_read(self, s_arg=None):
-        if s_arg is not None:
-            assert isinstance(s_arg, SomeInteger)
-        return SomeString(can_be_None=False)
+def llexternal(*args):
+    return rffi.llexternal(*args, compilation_info=eci)
 
-    def method_close(self):
-        pass
+FILE = lltype.Struct('FILE') # opaque type maybe
 
-    def method_seek(self, s_arg, s_whence=None):
-        assert isinstance(s_arg, SomeInteger)
-        if s_whence is not None:
-            assert isinstance(s_whence, SomeInteger)
+class CConfig(object):
+    _compilation_info_ = eci
 
-    def rtyper_makekey(self):
-        return self.__class__,
+    off_t = platform.SimpleType('off_t')
 
-    def rtyper_makerepr(self, rtyper):
-        from rpython.rtyper.lltypesystem.rfile import FileRepr
+CC = platform.configure(CConfig)
+OFF_T = CC['off_t']
 
-        return FileRepr(rtyper)
+c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
+c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
+c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
+                                     lltype.Ptr(FILE)], rffi.SIZE_T)
+c_read = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
+                                   lltype.Ptr(FILE)], rffi.SIZE_T)
+c_feof = llexternal('feof', [lltype.Ptr(FILE)], rffi.INT)
+c_ferror = llexternal('ferror', [lltype.Ptr(FILE)], rffi.INT)
+c_clearerror = llexternal('clearerr', [lltype.Ptr(FILE)], lltype.Void)
+c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT],
+                          rffi.INT)
+c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
+c_fileno = llexternal('fileno', [lltype.Ptr(FILE)], rffi.INT)
+c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], lltype.Signed)
+c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT)
+c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT)
+c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, lltype.Ptr(FILE)],
+                     rffi.CCHARP)
 
-class FileEntry(ExtRegistryEntry):
-    _about_ = open
+BASE_BUF_SIZE = 4096
+BASE_LINE_SIZE = 100
 
-    def compute_result_annotation(self, s_name, s_mode=None):
-        assert isinstance(s_name, SomeString)
-        if s_mode is not None:
-            assert isinstance(s_mode, SomeString)
-        return SomeFile()
+def create_file(filename, mode="r", buffering=-1):
+    assert buffering == -1
+    assert filename is not None
+    assert mode is not None
+    ll_name = rffi.str2charp(filename)
+    try:
+        ll_mode = rffi.str2charp(mode)
+        try:
+            ll_f = c_open(ll_name, ll_mode)
+            if not ll_f:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+        finally:
+            lltype.free(ll_mode, flavor='raw')
+    finally:
+        lltype.free(ll_name, flavor='raw')
+    return RFile(ll_f)
 
-    def specialize_call(self, hop):
-        return hop.r_result.rtype_constructor(hop)
+def create_temp_rfile():
+    res = c_tmpfile()
+    if not res:
+        errno = rposix.get_errno()
+        raise OSError(errno, os.strerror(errno))
+    return RFile(res)
 
-class OSTempfileEntry(ExtRegistryEntry):
-    _about_ = os.tmpfile
+class RFile(object):
+    def __init__(self, ll_file):
+        self.ll_file = ll_file
 
-    def compute_result_annotation(self):
-        return SomeFile()
+    def write(self, value):
+        assert value is not None
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        assert value is not None
+        ll_value = rffi.get_nonmovingbuffer(value)
+        try:
+            # note that since we got a nonmoving buffer, it is either raw
+            # or already cannot move, so the arithmetics below are fine
+            total_bytes = 0
+            ll_current = ll_value
+            while total_bytes < len(value):
+                bytes = c_write(ll_current, 1, len(value) - r_uint(total_bytes),
+                                ll_file)
+                if bytes == 0:
+                    errno = rposix.get_errno()
+                    raise OSError(errno, os.strerror(errno))
+                total_bytes += bytes
+                ll_current = rffi.cast(rffi.CCHARP,
+                                       rffi.cast(lltype.Unsigned, ll_value) +
+                                       total_bytes)
+        finally:
+            rffi.free_nonmovingbuffer(value, ll_value)
 
-    def specialize_call(self, hop):
-        return hop.r_result.rtype_tempfile(hop)
+    def close(self):
+        if self.ll_file:
+            # double close is allowed
+            res = c_close(self.ll_file)
+            self.ll_file = lltype.nullptr(FILE)
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
 
+    def read(self, size=-1):
+        # XXX CPython uses a more delicate logic here
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        if size < 0:
+            # read the entire contents
+            buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
+            try:
+                s = StringBuilder()
+                while True:
+                    returned_size = c_read(buf, 1, BASE_BUF_SIZE, ll_file)
+                    if returned_size == 0:
+                        if c_feof(ll_file):
+                            # ok, finished
+                            return s.build()
+                        errno = c_ferror(ll_file)
+                        c_clearerror(ll_file)
+                        raise OSError(errno, os.strerror(errno))
+                    s.append_charpsize(buf, returned_size)
+            finally:
+                lltype.free(buf, flavor='raw')
+        else:
+            raw_buf, gc_buf = rffi.alloc_buffer(size)
+            try:
+                returned_size = c_read(raw_buf, 1, size, ll_file)
+                if returned_size == 0:
+                    if not c_feof(ll_file):
+                        errno = c_ferror(ll_file)
+                        raise OSError(errno, os.strerror(errno))
+                s = rffi.str_from_buffer(raw_buf, gc_buf, size,
+                                         rffi.cast(lltype.Signed, returned_size))
+            finally:
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+            return s
+
+    def seek(self, pos, whence=0):
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        res = c_fseek(ll_file, pos, whence)
+        if res == -1:
+            errno = rposix.get_errno()
+            raise OSError(errno, os.strerror(errno))
+
+    def fileno(self):
+        if self.ll_file:
+            return intmask(c_fileno(self.ll_file))
+        raise ValueError("I/O operation on closed file")
+
+    def tell(self):
+        if self.ll_file:
+            res = intmask(c_ftell(self.ll_file))
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return res
+        raise ValueError("I/O operation on closed file")
+
+    def flush(self):
+        if self.ll_file:
+            res = c_fflush(self.ll_file)
+            if res != 0:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return
+        raise ValueError("I/O operation on closed file")
+
+    def truncate(self, arg=-1):
+        if self.ll_file:
+            if arg == -1:
+                arg = self.tell()
+            res = c_ftruncate(self.fileno(), arg)
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return
+        raise ValueError("I/O operation on closed file")
+
+    def __del__(self):
+        self.close()
+
+    def _readline1(self, raw_buf):
+        result = c_fgets(raw_buf, BASE_LINE_SIZE, self.ll_file)
+        if not result:
+            if c_feof(self.ll_file):   # ok
+                return 0
+            errno = c_ferror(self.ll_file)
+            raise OSError(errno, os.strerror(errno))
+        #
+        # Assume that fgets() works as documented, and additionally
+        # never writes beyond the final \0, which the CPython
+        # fileobject.c says appears to be the case everywhere.
+        # The only case where the buffer was not big enough is the
+        # case where the buffer is full, ends with \0, and doesn't
+        # end with \n\0.
+        strlen = 0
+        while raw_buf[strlen] != '\0':
+            strlen += 1
+        if (strlen == BASE_LINE_SIZE - 1 and
+              raw_buf[BASE_LINE_SIZE - 2] != '\n'):
+            return -1    # overflow!
+        # common case
+        return strlen
+
+    def readline(self):
+        if self.ll_file:
+            raw_buf, gc_buf = rffi.alloc_buffer(BASE_LINE_SIZE)
+            try:
+                c = self._readline1(raw_buf)
+                if c >= 0:
+                    return rffi.str_from_buffer(raw_buf, gc_buf,
+                                                BASE_LINE_SIZE, c)
+                #
+                # this is the rare case: the line is longer than BASE_LINE_SIZE
+                s = StringBuilder()
+                while True:
+                    s.append_charpsize(raw_buf, BASE_LINE_SIZE - 1)
+                    c = self._readline1(raw_buf)
+                    if c >= 0:
+                        break
+                #
+                s.append_charpsize(raw_buf, c)
+                return s.build()
+            finally:
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+        raise ValueError("I/O operation on closed file")
diff --git a/rpython/rlib/rpath.py b/rpython/rlib/rpath.py
--- a/rpython/rlib/rpath.py
+++ b/rpython/rlib/rpath.py
@@ -61,3 +61,65 @@
     except os.error:
         return False
     return True
+
+
+import os
+from os.path import isabs, islink, abspath, normpath
+
+def join(a, p):
+    """Join two or more pathname components, inserting '/' as needed.
+    If any component is an absolute path, all previous path components
+    will be discarded.  An empty last part will result in a path that
+    ends with a separator."""
+    path = a
+    for b in p:
+        if b.startswith('/'):
+            path = b
+        elif path == '' or path.endswith('/'):
+            path +=  b
+        else:
+            path += '/' + b
+    return path
+
+def realpath(filename):
+    """Return the canonical path of the specified filename, eliminating any
+symbolic links encountered in the path."""
+    if isabs(filename):
+        bits = ['/'] + filename.split('/')[1:]
+    else:
+        bits = [''] + filename.split('/')
+
+    for i in range(2, len(bits)+1):
+        component = join(bits[0], bits[1:i])
+        # Resolve symbolic links.
+        if islink(component):
+            resolved = _resolve_link(component)
+            if resolved is None:
+                # Infinite loop -- return original component + rest of the path
+                return abspath(join(component, bits[i:]))
+            else:
+                newpath = join(resolved, bits[i:])
+                return realpath(newpath)
+
+    return abspath(filename)
+
+
+def _resolve_link(path):
+    """Internal helper function.  Takes a path and follows symlinks
+    until we either arrive at something that isn't a symlink, or
+    encounter a path we've seen before (meaning that there's a loop).
+    """
+    paths_seen = {}
+    while islink(path):
+        if path in paths_seen:
+            # Already seen this path, so we must have a symlink loop
+            return None
+        paths_seen[path] = None
+        # Resolve where the link points to
+        resolved = os.readlink(path)
+        if not isabs(resolved):
+            dir = dirname(path)
+            path = normpath(join(dir, [resolved]))
+        else:
+            path = normpath(resolved)
+    return path
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -78,3 +78,100 @@
 
         f()
         self.interpret(f, [])
+
+    def test_fileno(self):
+        fname = str(self.tmpdir.join('file_5'))
+
+        def f():
+            f = open(fname, "w")
+            try:
+                return f.fileno()
+            finally:
+                f.close()
+
+        res = self.interpret(f, [])
+        assert res > 2
+
+    def test_tell(self):
+        fname = str(self.tmpdir.join('file_tell'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            try:
+                return f.tell()
+            finally:
+                f.close()
+
+        res = self.interpret(f, [])
+        assert res == 3
+
+    def test_flush(self):
+        fname = str(self.tmpdir.join('file_flush'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            f.flush()
+            f2 = open(fname)
+            assert f2.read() == "xyz"
+            f2.close()
+            f.close()
+
+        self.interpret(f, [])
+
+    def test_truncate(self):
+        fname = str(self.tmpdir.join('file_trunc'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            f.seek(0)
+            f.truncate(2)
+            f.close()
+            f2 = open(fname)
+            assert f2.read() == "xy"
+            f2.close()
+
+        f()
+        self.interpret(f, [])
+
+
+class TestDirect:
+    def setup_class(cls):
+        cls.tmpdir = udir.join('test_rfile_direct')
+        cls.tmpdir.ensure(dir=True)
+
+    def test_readline(self):
+        fname = str(self.tmpdir.join('file_readline'))
+        j = 0
+        expected = []
+        with open(fname, 'w') as f:
+            for i in range(250):
+                s = ''.join([chr(32+(k&63)) for k in range(j, j + i)])
+                j += 1
+                print >> f, s
+        expected = open(fname).readlines()
+        expected += ['', '']
+        assert len(expected) == 252
+
+        f = rfile.create_file(fname, 'r')
+        for j in range(252):
+            got = f.readline()
+            assert got == expected[j]
+        f.close()
+
+    def test_readline_without_eol_at_the_end(self):
+        fname = str(self.tmpdir.join('file_readline_without_eol_at_the_end'))
+        for n in [1, 10, 97, 98, 99, 100, 101, 102, 103, 150,
+                  196, 197, 198, 199, 200, 201, 202, 203, 204, 250]:
+            s = ''.join([chr(32+(k&63)) for k in range(n)])
+            with open(fname, 'wb') as f:
+                f.write(s)
+
+            f = rfile.create_file(fname, 'r')
+            got = f.readline()
+            assert got == s
+            got = f.readline()
+            assert got == ''
+            f.close()
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -301,6 +301,8 @@
             addr.get_port() == 80):
             found = True
     assert found, lst
+    # The following might fail if the DNS redirects failed requests to a
+    # catch-all address (i.e. opendns).
     e = py.test.raises(GAIError, getaddrinfo, 'www.very-invalidaddress.com', None)
     assert isinstance(e.value.get_msg(), str)
 
diff --git a/rpython/rlib/test/test_rstacklet.py b/rpython/rlib/test/test_rstacklet.py
--- a/rpython/rlib/test/test_rstacklet.py
+++ b/rpython/rlib/test/test_rstacklet.py
@@ -1,5 +1,6 @@
 import gc, sys
 import py
+import platform
 from rpython.rtyper.tool.rffi_platform import CompilationError
 try:
     from rpython.rlib import rstacklet
@@ -332,6 +333,10 @@
     gc = 'minimark'
     gcrootfinder = 'asmgcc'
 
+    @py.test.mark.skipif("sys.platform != 'linux2' or platform.machine().startswith('arm')")
+    def test_demo1(self):
+        BaseTestStacklet.test_demo1(self)
+
 class TestStackletShadowStack(BaseTestStacklet):
     gc = 'minimark'
     gcrootfinder = 'shadowstack'
diff --git a/rpython/rlib/types.py b/rpython/rlib/types.py
--- a/rpython/rlib/types.py
+++ b/rpython/rlib/types.py
@@ -39,8 +39,12 @@
     return model.SomeUnicodeString(no_nul=True)
 
 
-def str():
-    return model.SomeString()
+def str(can_be_None=False):
+    return model.SomeString(can_be_None=can_be_None)
+
+
+def bytearray():
+    return model.SomeByteArray()
 
 
 def str0():
diff --git a/rpython/rtyper/lltypesystem/rbytearray.py b/rpython/rtyper/lltypesystem/rbytearray.py
--- a/rpython/rtyper/lltypesystem/rbytearray.py
+++ b/rpython/rtyper/lltypesystem/rbytearray.py
@@ -16,6 +16,9 @@
                                                                   lltype.Char,
                                                                   'bytearray_from_str')
 
+def _empty_bytearray():
+    return empty
+
 BYTEARRAY.become(lltype.GcStruct('rpy_bytearray',
                  ('chars', lltype.Array(lltype.Char)), adtmeths={
     'malloc' : lltype.staticAdtMethod(mallocbytearray),
@@ -23,8 +26,11 @@
     'copy_contents_from_str': lltype.staticAdtMethod(
                                          copy_bytearray_contents_from_str),
     'length': rstr.LLHelpers.ll_length,
+    'empty': lltype.staticAdtMethod(_empty_bytearray),
 }))
 
+empty = lltype.malloc(BYTEARRAY, 0, immortal=True)
+
 class LLHelpers(rstr.LLHelpers):
     @classmethod
     def ll_strsetitem(cls, s, i, item):
diff --git a/rpython/rtyper/lltypesystem/rfile.py b/rpython/rtyper/lltypesystem/rfile.py
deleted file mode 100644
--- a/rpython/rtyper/lltypesystem/rfile.py
+++ /dev/null
@@ -1,195 +0,0 @@
-
-import os
-from rpython.rlib import rposix
-from rpython.rlib.rarithmetic import r_uint
-from rpython.annotator import model as annmodel
-from rpython.rtyper.rtyper import Repr
-from rpython.rlib.rstring import StringBuilder
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
-from rpython.rtyper.lltypesystem.rstr import string_repr, STR
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.annlowlevel import hlstr
-from rpython.rtyper.lltypesystem.lloperation import llop
-
-FILE = lltype.Struct('FILE') # opaque type maybe
-FILE_WRAPPER = lltype.GcStruct("FileWrapper", ('file', lltype.Ptr(FILE)))
-
-eci = ExternalCompilationInfo(includes=['stdio.h'])
-
-def llexternal(*args):
-    return rffi.llexternal(*args, compilation_info=eci)
-
-c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
-c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
-c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
-                                     lltype.Ptr(FILE)], rffi.SIZE_T)
-c_read = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
-                                   lltype.Ptr(FILE)], rffi.SIZE_T)
-c_feof = llexternal('feof', [lltype.Ptr(FILE)], rffi.INT)
-c_ferror = llexternal('ferror', [lltype.Ptr(FILE)], rffi.INT)
-c_clearerror = llexternal('clearerr', [lltype.Ptr(FILE)], lltype.Void)
-c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT],
-                          rffi.INT)
-c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
-
-def ll_open(name, mode):
-    file_wrapper = lltype.malloc(FILE_WRAPPER)
-    ll_name = rffi.str2charp(name)
-    ll_mode = rffi.str2charp(mode)
-    try:
-        ll_f = c_open(ll_name, ll_mode)
-        if not ll_f:
-            errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
-        file_wrapper.file = ll_f
-    finally:
-        lltype.free(ll_name, flavor='raw')
-        lltype.free(ll_mode, flavor='raw')
-    return file_wrapper
-
-def ll_tmpfile():
-    file_wrapper = lltype.malloc(FILE_WRAPPER)
-    res = c_tmpfile()
-    if not res:
-        errno = rposix.get_errno()
-        raise OSError(errno, os.strerror(errno))
-    file_wrapper.file = res
-    return file_wrapper
-
-def ll_write(file_wrapper, value):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    value = hlstr(value)
-    assert value is not None
-    ll_value = rffi.get_nonmovingbuffer(value)
-    try:
-        # note that since we got a nonmoving buffer, it is either raw
-        # or already cannot move, so the arithmetics below are fine
-        total_bytes = 0
-        ll_current = ll_value
-        while total_bytes < len(value):
-            bytes = c_write(ll_current, 1, len(value) - r_uint(total_bytes),
-                            ll_file)
-            if bytes == 0:
-                errno = rposix.get_errno()
-                raise OSError(errno, os.strerror(errno))
-            total_bytes += bytes
-            ll_current = rffi.cast(rffi.CCHARP,
-                                   rffi.cast(lltype.Unsigned, ll_value) +
-                                   total_bytes)
-    finally:
-        rffi.free_nonmovingbuffer(value, ll_value)
-
-BASE_BUF_SIZE = 4096
-
-def ll_read(file_wrapper, size):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    if size < 0:
-        # read the entire contents
-        buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
-        try:
-            s = StringBuilder()
-            while True:
-                returned_size = c_read(buf, 1, BASE_BUF_SIZE, ll_file)
-                if returned_size == 0:
-                    if c_feof(ll_file):
-                        # ok, finished
-                        return s.build()
-                    errno = c_ferror(ll_file)
-                    c_clearerror(ll_file)
-                    raise OSError(errno, os.strerror(errno))
-                s.append_charpsize(buf, returned_size)
-        finally:
-            lltype.free(buf, flavor='raw')
-    else:
-        raw_buf, gc_buf = rffi.alloc_buffer(size)
-        try:
-            returned_size = c_read(raw_buf, 1, size, ll_file)
-            if returned_size == 0:
-                if not c_feof(ll_file):
-                    errno = c_ferror(ll_file)
-                    raise OSError(errno, os.strerror(errno))
-            s = rffi.str_from_buffer(raw_buf, gc_buf, size,
-                                     rffi.cast(lltype.Signed, returned_size))
-        finally:
-            rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
-        return s
-def ll_seek(file_wrapper, pos, whence):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    res = c_fseek(ll_file, pos, whence)
-    if res == -1:
-        errno = rposix.get_errno()
-        raise OSError(errno, os.strerror(errno))
-
-def ll_close(file_wrapper):
-    if file_wrapper.file:
-        # double close is allowed
-        res = c_close(file_wrapper.file)
-        file_wrapper.file = lltype.nullptr(FILE)
-        if res == -1:
-            errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
-
-class FileRepr(Repr):
-    lowleveltype = lltype.Ptr(FILE_WRAPPER)
-
-    def __init__(self, typer):
-        Repr.__init__(self)
-
-    def rtype_constructor(self, hop):
-        repr = hop.rtyper.getrepr(annmodel.SomeString())
-        arg_0 = hop.inputarg(repr, 0)
-        if len(hop.args_v) == 1:
-            arg_1 = hop.inputconst(string_repr, "r")
-        else:
-            arg_1 = hop.inputarg(repr, 1)
-        hop.exception_is_here()
-        open = hop.rtyper.getannmixlevel().delayedfunction(
-            ll_open, [annmodel.SomeString()] * 2,
-            annmodel.SomePtr(self.lowleveltype))
-        v_open = hop.inputconst(lltype.typeOf(open), open)
-        return hop.genop('direct_call', [v_open, arg_0, arg_1],
-                         resulttype=self)
-
-    def rtype_tempfile(self, hop):
-        tmpfile = hop.rtyper.getannmixlevel().delayedfunction(
-            ll_tmpfile, [], annmodel.SomePtr(self.lowleveltype))
-        v_tmpfile = hop.inputconst(lltype.typeOf(tmpfile), tmpfile)
-        hop.exception_is_here()
-        return hop.genop('direct_call', [v_tmpfile], resulttype=self)
-
-
-    def rtype_method_write(self, hop):
-        args_v = hop.inputargs(self, string_repr)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_write, *args_v)
-
-    def rtype_method_close(self, hop):
-        r_self = hop.inputarg(self, 0)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_close, r_self)
-
-    def rtype_method_read(self, hop):
-        r_self = hop.inputarg(self, 0)
-        if len(hop.args_v) != 2:
-            arg_1 = hop.inputconst(lltype.Signed, -1)
-        else:
-            arg_1 = hop.inputarg(lltype.Signed, 1)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_read, r_self, arg_1)
-
-    def rtype_method_seek(self, hop):
-        r_self = hop.inputarg(self, 0)
-        arg_1 = hop.inputarg(lltype.Signed, 1)
-        if len(hop.args_v) != 3:
-            arg_2 = hop.inputconst(lltype.Signed, os.SEEK_SET)
-        else:
-            arg_2 = hop.inputarg(lltype.Signed, 2)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_seek, r_self, arg_1, arg_2)
-
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -1685,7 +1685,7 @@
         def tmpnam_llimpl():
             return rffi.charp2str(os_tmpnam(lltype.nullptr(rffi.CCHARP.TO)))
 
-        return extdef([], None, llimpl=tmpnam_llimpl,
+        return extdef([], str, llimpl=tmpnam_llimpl,
                       export_name="ll_os.ll_os_tmpnam")
 
 # --------------------------- os.stat & variants ---------------------------
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -187,13 +187,15 @@
         if hop.nb_args > 2:
             v_start = hop.inputarg(Signed, arg=2)
             if not hop.args_s[2].nonneg:
-                raise TyperError("str.find() start must be proven non-negative")
+                raise TyperError("str.%s() start must be proven non-negative"
+                                 % (reverse and 'rfind' or 'find',))
         else:
             v_start = hop.inputconst(Signed, 0)
         if hop.nb_args > 3:
             v_end = hop.inputarg(Signed, arg=3)
             if not hop.args_s[3].nonneg:
-                raise TyperError("str.find() end must be proven non-negative")
+                raise TyperError("str.%s() end must be proven non-negative"
+                                 % (reverse and 'rfind' or 'find',))
         else:
             v_end = hop.gendirectcall(self.ll.ll_strlen, v_str)
         hop.exception_cannot_occur()
diff --git a/rpython/rtyper/test/test_rbytearray.py b/rpython/rtyper/test/test_rbytearray.py
--- a/rpython/rtyper/test/test_rbytearray.py
+++ b/rpython/rtyper/test/test_rbytearray.py
@@ -50,3 +50,10 @@
 
         ll_res = self.interpret(f, [123])
         assert hlstr(ll_res) == "123"
+
+    def test_getslice(self):
+        def f(x):
+            return str(bytearray(str(x))[1:2])
+
+        ll_res = self.interpret(f, [123])
+        assert hlstr(ll_res) == "2"
diff --git a/rpython/rtyper/test/test_rint.py b/rpython/rtyper/test/test_rint.py
--- a/rpython/rtyper/test/test_rint.py
+++ b/rpython/rtyper/test/test_rint.py
@@ -85,6 +85,14 @@
         res = self.ll_to_string(res)
         assert res == '-0x8' + '0' * (len(res)-4)
 
+    def test_hex_of_uint(self):
+        def dummy(i):
+            return hex(r_uint(i))
+
+        res = self.interpret(dummy, [-5])
+        res = self.ll_to_string(res)
+        assert res == '0x' + 'f' * (len(res)-3) + 'b'
+
     def test_oct_of_int(self):
         def dummy(i):
             return oct(i)
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -416,6 +416,14 @@
             res = self.interpret(f, [i])
             assert res == expected
 
+    def test_rfind_error_message(self):
+        const = self.const
+        def f(i):
+            return const("abc").rfind(const(''), i)
+        e = py.test.raises(TyperError, self.interpret, f, [-5])
+        assert str(e.value).startswith(
+            'str.rfind() start must be proven non-negative')
+
     def test_find_char(self):
         const = self.const
         def fn(ch):
@@ -1134,4 +1142,4 @@
         array = lltype.malloc(TP, 12, flavor='raw')
         self.interpret(f, [array, 4])
         assert list(array) == list('abc'*4)
-        lltype.free(array, flavor='raw')
\ No newline at end of file
+        lltype.free(array, flavor='raw')
diff --git a/rpython/translator/c/src/stacklet/switch_arm_gcc.h b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
--- a/rpython/translator/c/src/stacklet/switch_arm_gcc.h
+++ b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
@@ -1,9 +1,8 @@
-#if __ARM_ARCH__ >= 5
-# define call_reg(x) "blx " #x "\n"
-#elif defined (__ARM_ARCH_4T__)
+#if defined(__ARM_ARCH_4__) || defined (__ARM_ARCH_4T__)
 # define call_reg(x) "mov lr, pc ; bx " #x "\n"
 #else
-# define call_reg(x) "mov lr, pc ; mov pc, " #x "\n"
+/* ARM >= 5 */
+# define call_reg(x) "blx " #x "\n"
 #endif
 
 static void __attribute__((optimize("O3"))) *slp_switch(void *(*save_state)(void*, void*),
diff --git a/rpython/translator/tool/graphpage.py b/rpython/translator/tool/graphpage.py
--- a/rpython/translator/tool/graphpage.py
+++ b/rpython/translator/tool/graphpage.py
@@ -200,7 +200,7 @@
                 dotgen.emit_edge(nameof(cdef), nameof(prevcdef), color="red")
             prevcdef = cdef
             cdef = cdef.basedef
-        
+
         self.source = dotgen.generate(target=None)
 
     def followlink(self, name):
@@ -224,7 +224,7 @@
         dotgen.emit('mclimit=15.0')
 
         self.do_compute(dotgen, *args, **kwds)
-        
+
         self.source = dotgen.generate(target=None)
 
         # link the function names to the individual flow graphs
@@ -264,7 +264,7 @@
                 data = self.labelof(classdef, classdef.shortname)
                 dotgen.emit_node(nameof(classdef), label=data, shape="box")
                 dotgen.emit_edge(nameof(classdef.basedef), nameof(classdef))
-             
+
     def labelof(self, obj, objname):
         name = objname
         i = 1
@@ -409,22 +409,11 @@
     elif isinstance(obj, Link):
         try_show(obj.prevblock)
     elif isinstance(obj, Block):
-        import gc
-        pending = [obj]   # pending blocks
-        seen = {obj: True, None: True}
-        for x in pending:
-            for y in gc.get_referrers(x):
-                if isinstance(y, FunctionGraph):
-                    y.show()
-                    return
-                elif isinstance(y, Link):
-                    block = y.prevblock
-                    if block not in seen:
-                        pending.append(block)
-                        seen[block] = True
-                elif isinstance(y, dict):
-                    pending.append(y)   # go back from the dict to the real obj
-        graph = IncompleteGraph(pending)
+        graph = obj.get_graph()
+        if isinstance(graph, FunctionGraph):
+            graph.show()
+            return
+        graph = IncompleteGraph(graph)
         SingleGraphPage(graph).display()
     else:
         raise TypeError("try_show(%r object)" % (type(obj).__name__,))
@@ -449,7 +438,7 @@
                         seen[block] = True
         return pending
     else:
-        raise TypeError("try_get_functiongraph(%r object)" % (type(obj).__name__,))    
+        raise TypeError("try_get_functiongraph(%r object)" % (type(obj).__name__,))
 
 class IncompleteGraph:
     name = '(incomplete graph)'

From noreply at buildbot.pypy.org  Thu Oct 10 00:45:57 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 00:45:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: adapt get_include() to function after
 installing numpy to site-packages
Message-ID: <20131009224557.404D81C33FE@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67269:e37ed16ca2e0
Date: 2013-10-10 01:44 +0300
http://bitbucket.org/pypy/pypy/changeset/e37ed16ca2e0/

Log:	adapt get_include() to function after installing numpy to site-
	packages

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -10,8 +10,35 @@
 
 import os
 def get_include():
-    head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.join(head, '../include')
+    """
+    Return the directory that contains the NumPy \\*.h header files.
+
+    Extension modules that need to compile against NumPy should use this
+    function to locate the appropriate include directory.
+
+    Notes
+    -----
+    When using ``distutils``, for example in ``setup.py``.
+    ::
+
+        import numpy as np
+        ...
+        Extension('extension_name', ...
+                include_dirs=[np.get_include()])
+        ...
+
+    """
+    import numpy
+    if getattr(numpy, 'show_config', None) is None:
+        # running from numpy source directory
+        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
+        return os.path.join(head, '../include')
+    else:
+        # using installed numpy core headers
+        import numpy.core as core
+        d = os.path.join(os.path.dirname(core.__file__), 'include')
+    return d
+
 
 
 __all__ = ['__version__', 'get_include']

From noreply at buildbot.pypy.org  Thu Oct 10 01:22:22 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 10 Oct 2013 01:22:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: pull out a special case for py3k which
 returns complex in this case
Message-ID: <20131009232222.0B5E41C36AD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r67270:09045c74ae0c
Date: 2013-10-09 16:10 -0700
http://bitbucket.org/pypy/pypy/changeset/09045c74ae0c/

Log:	pull out a special case for py3k which returns complex in this case

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -1,5 +1,5 @@
 import operator
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.objspace.std import model, newformat
 from pypy.objspace.std.floattype import float_typedef, W_AbstractFloatObject
 from pypy.objspace.std.multimethod import FailedToImplementArgs
@@ -424,7 +424,16 @@
     x = w_float1.floatval
     y = w_float2.floatval
 
-    return W_FloatObject(_pow(space, x, y))
+    try:
+        result = _pow(space, x, y)
+    except PowDomainError:
+        raise operationerrfmt(space.w_ValueError,
+                             "negative number cannot be raised to a "
+                              "fractional power")
+    return W_FloatObject(result)
+
+class PowDomainError(ValueError):
+    """Signals a negative number raised to a fractional power"""
 
 def _pow(space, x, y):
     # Sort out special cases here instead of relying on pow()
@@ -478,16 +487,14 @@
                                             "a negative power"))
 
     negate_result = False
-    # special case: "(-1.0) ** bignum" should not raise ValueError,
+    # special case: "(-1.0) ** bignum" should not raise PowDomainError,
     # unlike "math.pow(-1.0, bignum)".  See http://mail.python.org/
     # -           pipermail/python-bugs-list/2003-March/016795.html
     if x < 0.0:
         if isnan(y):
             return NAN
         if math.floor(y) != y:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("negative number cannot be "
-                                            "raised to a fractional power"))
+            raise PowDomainError
         # y is an exact integer, albeit perhaps a very large one.
         # Replace x by its absolute value and remember to negate the
         # pow result if y is odd.

From noreply at buildbot.pypy.org  Thu Oct 10 01:22:23 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 10 Oct 2013 01:22:23 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20131009232223.7F8F71C36AD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67271:8dbb46471ccd
Date: 2013-10-09 16:18 -0700
http://bitbucket.org/pypy/pypy/changeset/8dbb46471ccd/

Log:	merge default

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -10,8 +10,35 @@
 
 import os
 def get_include():
-    head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.join(head, '../include')
+    """
+    Return the directory that contains the NumPy \\*.h header files.
+
+    Extension modules that need to compile against NumPy should use this
+    function to locate the appropriate include directory.
+
+    Notes
+    -----
+    When using ``distutils``, for example in ``setup.py``.
+    ::
+
+        import numpy as np
+        ...
+        Extension('extension_name', ...
+                include_dirs=[np.get_include()])
+        ...
+
+    """
+    import numpy
+    if getattr(numpy, 'show_config', None) is None:
+        # running from numpy source directory
+        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
+        return os.path.join(head, '../include')
+    else:
+        # using installed numpy core headers
+        import numpy.core as core
+        d = os.path.join(os.path.dirname(core.__file__), 'include')
+    return d
+
 
 
 __all__ = ['__version__', 'get_include']
diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -1,1 +1,75 @@
 from _numpypy.numerictypes import *
+import numpypy
+
+def issubclass_(arg1, arg2):
+    """
+    Determine if a class is a subclass of a second class.
+
+    `issubclass_` is equivalent to the Python built-in ``issubclass``,
+    except that it returns False instead of raising a TypeError is one
+    of the arguments is not a class.
+
+    Parameters
+    ----------
+    arg1 : class
+        Input class. True is returned if `arg1` is a subclass of `arg2`.
+    arg2 : class or tuple of classes.
+        Input class. If a tuple of classes, True is returned if `arg1` is a
+        subclass of any of the tuple elements.
+
+    Returns
+    -------
+    out : bool
+        Whether `arg1` is a subclass of `arg2` or not.
+
+    See Also
+    --------
+    issubsctype, issubdtype, issctype
+
+    Examples
+    --------
+    >>> np.issubclass_(np.int32, np.int)
+    True
+    >>> np.issubclass_(np.int32, np.float)
+    False
+
+    """
+    try:
+        return issubclass(arg1, arg2)
+    except TypeError:
+        return False
+
+def issubdtype(arg1, arg2):
+    """
+    Returns True if first argument is a typecode lower/equal in type hierarchy.
+
+    Parameters
+    ----------
+    arg1, arg2 : dtype_like
+        dtype or string representing a typecode.
+
+    Returns
+    -------
+    out : bool
+
+    See Also
+    --------
+    issubsctype, issubclass_
+    numpy.core.numerictypes : Overview of numpy type hierarchy.
+
+    Examples
+    --------
+    >>> np.issubdtype('S1', str)
+    True
+    >>> np.issubdtype(np.float64, np.float32)
+    False
+
+    """
+    if issubclass_(arg2, generic):
+        return issubclass(numpypy.dtype(arg1).type, arg2)
+    mro = numpypy.dtype(arg2).type.mro()
+    if len(mro) > 1:
+        val = mro[1]
+    else:
+        val = mro[0]
+    return issubclass(numpypy.dtype(arg1).type, val)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -395,6 +395,8 @@
     def test_socket_connect_ex(self):
         import _socket
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+        # The following might fail if the DNS redirects failed requests to a
+        # catch-all address (i.e. opendns).
         # Make sure we get an app-level error, not an interp one.
         raises(_socket.gaierror, s.connect_ex, ("wrong.invalid", 80))
         s.close()
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -99,11 +99,13 @@
     exc_p[0] = make_ref(space, operr.w_type)
     val_p[0] = make_ref(space, operr.get_w_value(space))
 
- at cpython_api([], lltype.Void)
+ at cpython_api([], rffi.INT_real, error=0)
 def PyErr_BadArgument(space):
     """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where
     message indicates that a built-in operation was invoked with an illegal
-    argument.  It is mostly for internal use."""
+    argument.  It is mostly for internal use. In CPython this function always
+    raises an exception and returns 0 in all cases, hence the (ab)use of the
+    error indicator."""
     raise OperationError(space.w_TypeError,
             space.wrap("bad argument type for built-in operation"))
 
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -61,9 +61,10 @@
         api.PyErr_Clear()
 
     def test_BadArgument(self, space, api):
-        api.PyErr_BadArgument()
+        ret = api.PyErr_BadArgument()
         state = space.fromcache(State)
         assert space.eq_w(state.operror.w_type, space.w_TypeError)
+        assert ret == 0
         api.PyErr_Clear()
 
     def test_Warning(self, space, api, capfd):
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -435,6 +435,15 @@
         rffi.free_charp(b_text)
         rffi.free_charp(b_encoding)
 
+    def test_decode_null_encoding(self, space, api):
+        null_charp = lltype.nullptr(rffi.CCHARP.TO)
+        u_text = u'abcdefg'
+        s_text = space.str_w(api.PyUnicode_AsEncodedString(space.wrap(u_text), null_charp, null_charp))
+        b_text = rffi.str2charp(s_text)
+        assert space.unwrap(api.PyUnicode_Decode(b_text, len(s_text), null_charp, null_charp)) == u_text
+        self.raises(space, api, TypeError, api.PyUnicode_FromEncodedObject, space.wrap(u_text), null_charp, None)
+        rffi.free_charp(b_text)
+
     def test_leak(self):
         size = 50
         raw_buf, gc_buf = rffi.alloc_buffer(size)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -354,6 +354,9 @@
     in the unicode() built-in function.  The codec to be used is looked up
     using the Python codec registry.  Return NULL if an exception was raised by
     the codec."""
+    if not encoding:
+        # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead
+        encoding = PyUnicode_GetDefaultEncoding(space)
     w_str = space.wrapbytes(rffi.charpsize2str(s, size))
     w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
@@ -383,6 +386,9 @@
 
     All other objects, including Unicode objects, cause a TypeError to be
     set."""
+    if not encoding:
+        raise OperationError(space.w_TypeError,
+                             space.wrap("decoding Unicode is not supported"))
     w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -319,6 +319,15 @@
         else:
             self.done_func = None
 
+    def are_common_types(self, dtype1, dtype2):
+        if dtype1.is_complex_type() and dtype2.is_complex_type():
+            return True
+        elif not (dtype1.is_complex_type() or dtype2.is_complex_type()) and \
+                (dtype1.is_int_type() and dtype2.is_int_type() or dtype1.is_float_type() and dtype2.is_float_type()) and \
+                not (dtype1.is_bool_type() or dtype2.is_bool_type()):
+            return True
+        return False
+
     @jit.unroll_safe
     def call(self, space, args_w):
         if len(args_w) > 2:
@@ -339,6 +348,12 @@
                  'unsupported operand dtypes %s and %s for "%s"' % \
                  (w_rdtype.get_name(), w_ldtype.get_name(),
                   self.name)))
+
+        if self.are_common_types(w_ldtype, w_rdtype):
+            if not w_lhs.is_scalar() and w_rhs.is_scalar():
+                w_rdtype = w_ldtype
+            elif w_lhs.is_scalar() and not w_rhs.is_scalar():
+                w_ldtype = w_rdtype
         calc_dtype = find_binop_result_dtype(space,
             w_ldtype, w_rdtype,
             int_only=self.int_only,
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2972,6 +2972,11 @@
                         dtype=[('bg', 'i8'), ('fg', 'i8'), ('char', 'S1')])
         assert c[0][0]["char"] == 'a'
 
+    def test_scalar_coercion(self):
+        import numpypy as np
+        a = np.array([1,2,3], dtype=np.int16)
+        assert (a * 2).dtype == np.int16
+
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -147,6 +147,8 @@
         interpleveldefs['nice'] = 'interp_posix.nice'
     if hasattr(os, 'getlogin'):
         interpleveldefs['getlogin'] = 'interp_posix.getlogin'
+    if hasattr(os, 'ctermid'):
+        interpleveldefs['ctermid'] = 'interp_posix.ctermid'
 
     for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid',
                  'seteuid', 'setgid', 'setegid', 'getgroups', 'getpgrp',
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1221,6 +1221,13 @@
     except OSError, e:
         raise wrap_oserror(space, e)
 
+def ctermid(space):
+    """ctermid() -> string
+
+    Return the name of the controlling terminal for this process.
+    """
+    return space.wrap(os.ctermid())
+
 @unwrap_spec(fd=int)
 def device_encoding(space, fd):
     """device_encoding(fd) -> str
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -110,7 +110,7 @@
                        '__pypy__', 'cStringIO', '_collections', 'struct',
                        'mmap', 'marshal', '_codecs', 'rctime', 'cppyy',
                        '_cffi_backend', 'pyexpat', '_continuation', '_io',
-                       'thread']:
+                       'thread', 'select']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -49,12 +49,15 @@
     from pypy.module.thread.os_lock import Lock
     assert pypypolicy.look_inside_function(Lock.descr_lock_acquire.im_func)
 
+def test_select():
+    from pypy.module.select.interp_select import poll
+    assert pypypolicy.look_inside_function(poll)
+
 def test_pypy_module():
     from pypy.module._collections.interp_deque import W_Deque
     from pypy.module._random.interp_random import W_Random
     assert not pypypolicy.look_inside_function(W_Random.random)
     assert pypypolicy.look_inside_function(W_Deque.length)
-    assert not pypypolicy.look_inside_pypy_module('select.interp_epoll')
     assert pypypolicy.look_inside_pypy_module('__builtin__.operation')
     assert pypypolicy.look_inside_pypy_module('__builtin__.abstractinst')
     assert pypypolicy.look_inside_pypy_module('__builtin__.functional')
diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -180,12 +180,12 @@
                         i = 0
                         for w_ev in space.listview(w_changelist):
                             ev = space.interp_w(W_Kevent, w_ev)
-                            changelist[i].c_ident = ev.event.c_ident
-                            changelist[i].c_filter = ev.event.c_filter
-                            changelist[i].c_flags = ev.event.c_flags
-                            changelist[i].c_fflags = ev.event.c_fflags
-                            changelist[i].c_data = ev.event.c_data
-                            changelist[i].c_udata = ev.event.c_udata
+                            changelist[i].c_ident = ev.ident
+                            changelist[i].c_filter = ev.filter
+                            changelist[i].c_flags = ev.flags
+                            changelist[i].c_fflags = ev.fflags
+                            changelist[i].c_data = ev.data
+                            changelist[i].c_udata = ev.udata
                             i += 1
                         pchangelist = changelist
                     else:
@@ -206,13 +206,12 @@
                             evt = eventlist[i]
 
                             w_event = W_Kevent(space)
-                            w_event.event = lltype.malloc(kevent, flavor="raw")
-                            w_event.event.c_ident = evt.c_ident
-                            w_event.event.c_filter = evt.c_filter
-                            w_event.event.c_flags = evt.c_flags
-                            w_event.event.c_fflags = evt.c_fflags
-                            w_event.event.c_data = evt.c_data
-                            w_event.event.c_udata = evt.c_udata
+                            w_event.ident = evt.c_ident
+                            w_event.filter = evt.c_filter
+                            w_event.flags = evt.c_flags
+                            w_event.fflags = evt.c_fflags
+                            w_event.data = evt.c_data
+                            w_event.udata = evt.c_udata
 
                             elist_w[i] = w_event
 
@@ -234,11 +233,12 @@
 
 class W_Kevent(W_Root):
     def __init__(self, space):
-        self.event = lltype.nullptr(kevent)
-
-    def __del__(self):
-        if self.event:
-            lltype.free(self.event, flavor="raw")
+        self.ident = rffi.cast(kevent.c_ident, 0)
+        self.filter = rffi.cast(kevent.c_filter, 0)
+        self.flags = rffi.cast(kevent.c_flags, 0)
+        self.fflags = rffi.cast(kevent.c_fflags, 0)
+        self.data = rffi.cast(kevent.c_data, 0)
+        self.udata = lltype.nullptr(rffi.VOIDP.TO)
 
     @unwrap_spec(filter=int, flags='c_uint', fflags='c_uint', data=int, udata=r_uint)
     def descr__init__(self, space, w_ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=r_uint(0)):
@@ -247,35 +247,34 @@
         else:
             ident = r_uint(space.c_filedescriptor_w(w_ident))
 
-        self.event = lltype.malloc(kevent, flavor="raw")
-        rffi.setintfield(self.event, "c_ident", ident)
-        rffi.setintfield(self.event, "c_filter", filter)
-        rffi.setintfield(self.event, "c_flags", flags)
-        rffi.setintfield(self.event, "c_fflags", fflags)
-        rffi.setintfield(self.event, "c_data", data)
-        self.event.c_udata = rffi.cast(rffi.VOIDP, udata)
+        self.ident = rffi.cast(kevent.c_ident, ident)
+        self.filter = rffi.cast(kevent.c_filter, filter)
+        self.flags = rffi.cast(kevent.c_flags, flags)
+        self.fflags = rffi.cast(kevent.c_fflags, fflags)
+        self.data = rffi.cast(kevent.c_data, data)
+        self.udata = rffi.cast(rffi.VOIDP, udata)
 
     def _compare_all_fields(self, other, op):
         if IDENT_UINT:
-            l_ident = rffi.cast(lltype.Unsigned, self.event.c_ident)
-            r_ident = rffi.cast(lltype.Unsigned, other.event.c_ident)
+            l_ident = rffi.cast(lltype.Unsigned, self.ident)
+            r_ident = rffi.cast(lltype.Unsigned, other.ident)
         else:
-            l_ident = self.event.c_ident
-            r_ident = other.event.c_ident
-        l_filter = rffi.cast(lltype.Signed, self.event.c_filter)
-        r_filter = rffi.cast(lltype.Signed, other.event.c_filter)
-        l_flags = rffi.cast(lltype.Unsigned, self.event.c_flags)
-        r_flags = rffi.cast(lltype.Unsigned, other.event.c_flags)
-        l_fflags = rffi.cast(lltype.Unsigned, self.event.c_fflags)
-        r_fflags = rffi.cast(lltype.Unsigned, other.event.c_fflags)
+            l_ident = self.ident
+            r_ident = other.ident
+        l_filter = rffi.cast(lltype.Signed, self.filter)
+        r_filter = rffi.cast(lltype.Signed, other.filter)
+        l_flags = rffi.cast(lltype.Unsigned, self.flags)
+        r_flags = rffi.cast(lltype.Unsigned, other.flags)
+        l_fflags = rffi.cast(lltype.Unsigned, self.fflags)
+        r_fflags = rffi.cast(lltype.Unsigned, other.fflags)
         if IDENT_UINT:
-            l_data = rffi.cast(lltype.Signed, self.event.c_data)
-            r_data = rffi.cast(lltype.Signed, other.event.c_data)
+            l_data = rffi.cast(lltype.Signed, self.data)
+            r_data = rffi.cast(lltype.Signed, other.data)
         else:
-            l_data = self.event.c_data
-            r_data = other.event.c_data
-        l_udata = rffi.cast(lltype.Unsigned, self.event.c_udata)
-        r_udata = rffi.cast(lltype.Unsigned, other.event.c_udata)
+            l_data = self.data
+            r_data = other.data
+        l_udata = rffi.cast(lltype.Unsigned, self.udata)
+        r_udata = rffi.cast(lltype.Unsigned, other.udata)
 
         if op == "eq":
             return l_ident == r_ident and \
@@ -330,22 +329,22 @@
         return space.wrap(self.compare_all_fields(space, w_other, "gt"))
 
     def descr_get_ident(self, space):
-        return space.wrap(self.event.c_ident)
+        return space.wrap(self.ident)
 
     def descr_get_filter(self, space):
-        return space.wrap(self.event.c_filter)
+        return space.wrap(self.filter)
 
     def descr_get_flags(self, space):
-        return space.wrap(self.event.c_flags)
+        return space.wrap(self.flags)
 
     def descr_get_fflags(self, space):
-        return space.wrap(self.event.c_fflags)
+        return space.wrap(self.fflags)
 
     def descr_get_data(self, space):
-        return space.wrap(self.event.c_data)
+        return space.wrap(self.data)
 
     def descr_get_udata(self, space):
-        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.event.c_udata))
+        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.udata))
 
 
 W_Kevent.typedef = TypeDef("select.kevent",
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -1,5 +1,5 @@
 import operator
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.objspace.std import model, newformat
 from pypy.objspace.std.floattype import float_typedef, W_AbstractFloatObject
 from pypy.objspace.std.multimethod import FailedToImplementArgs
@@ -398,9 +398,18 @@
     x = w_float1.floatval
     y = w_float2.floatval
 
-    return W_FloatObject(_pow(space, x, y, thirdArg))
+    try:
+        result = _pow(space, x, y)
+    except PowDomainError:
+        raise operationerrfmt(space.w_ValueError,
+                             "negative number cannot be raised to a "
+                              "fractional power")
+    return W_FloatObject(result)
 
-def _pow(space, x, y, thirdArg):
+class PowDomainError(ValueError):
+    """Signals a negative number raised to a fractional power"""
+
+def _pow(space, x, y):
     # Sort out special cases here instead of relying on pow()
     if y == 2.0:       # special case for performance:
         return x * x   # x * x is always correct
@@ -452,18 +461,14 @@
                                             "a negative power"))
 
     negate_result = False
-    # special case: "(-1.0) ** bignum" should not raise ValueError,
+    # special case: "(-1.0) ** bignum" should not raise PowDomainError,
     # unlike "math.pow(-1.0, bignum)".  See http://mail.python.org/
     # -           pipermail/python-bugs-list/2003-March/016795.html
     if x < 0.0:
         if isnan(y):
             return NAN
         if math.floor(y) != y:
-            # Negative numbers raised to fractional powers become
-            # complex
-            return space.pow(space.newcomplex(x, 0.0),
-                             space.newcomplex(y, 0.0),
-                             thirdArg)
+            raise PowDomainError
         # y is an exact integer, albeit perhaps a very large one.
         # Replace x by its absolute value and remember to negate the
         # pow result if y is odd.
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -30,7 +30,7 @@
 from rpython.rlib import debug, jit, rerased
 from rpython.rlib.listsort import make_timsort_class
 from rpython.rlib.objectmodel import (
-    instantiate, newlist_hint, resizelist_hint, specialize)
+    instantiate, newlist_hint, resizelist_hint, specialize, import_from_mixin)
 from rpython.tool.sourcetools import func_with_new_name
 
 __all__ = ['W_ListObject', 'make_range_list', 'make_empty_list_with_size']
@@ -1149,7 +1149,6 @@
 
 
 class AbstractUnwrappedStrategy(object):
-    _mixin_ = True
 
     def wrap(self, unwrapped):
         raise NotImplementedError
@@ -1308,7 +1307,6 @@
 
     def setslice(self, w_list, start, step, slicelength, w_other):
         assert slicelength >= 0
-        items = self.unerase(w_list.lstorage)
 
         if self is self.space.fromcache(ObjectListStrategy):
             w_other = w_other._temporarily_as_objects()
@@ -1320,6 +1318,7 @@
             w_list.setslice(start, step, slicelength, w_other_as_object)
             return
 
+        items = self.unerase(w_list.lstorage)
         oldsize = len(items)
         len2 = w_other.length()
         if step == 1:  # Support list resizing for non-extended slices
@@ -1435,7 +1434,9 @@
         self.unerase(w_list.lstorage).reverse()
 
 
-class ObjectListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class ObjectListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "object"
 
@@ -1468,7 +1469,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class IntegerListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class IntegerListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0
     _applevel_repr = "int"
 
@@ -1499,7 +1502,30 @@
         return self.unerase(w_list.lstorage)
 
 
-class FloatListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            l = self.unerase(w_list.lstorage)
+            other = w_other.getitems_int()
+            assert other is not None
+            l += other
+            return
+        return self._base_extend_from_list(w_list, w_other)
+
+
+    _base_setslice = setslice
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            storage = self.erase(w_other.getitems_int())
+            w_other = W_ListObject.from_storage_and_strategy(
+                    self.space, storage, self)
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
+class FloatListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0.0
     _applevel_repr = "float"
 
@@ -1527,7 +1553,9 @@
             l.reverse()
 
 
-class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class StringListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "str"
 
@@ -1558,7 +1586,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class UnicodeListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class UnicodeListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "unicode"
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -227,6 +227,15 @@
         l.setslice(0, 1, 2, W_ListObject(space, [w('a'), w(2), w(3)]))
         assert isinstance(l.strategy, ObjectListStrategy)
 
+    def test_setslice_int_range(self):
+        space = self.space
+        w = space.wrap
+        l = W_ListObject(space, [w(1), w(2), w(3)])
+        assert isinstance(l.strategy, IntegerListStrategy)
+        l.setslice(0, 1, 2, make_range_list(space, 5, 1, 4))
+        assert isinstance(l.strategy, IntegerListStrategy)
+
+
     def test_setslice_List(self):
         space = self.space
 
@@ -471,6 +480,12 @@
         l4 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3), self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         assert self.space.eq_w(l3, l4)
 
+    def test_add_of_range_and_int(self):
+        l1 = make_range_list(self.space, 0, 1, 100)
+        l2 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
+        l3 = self.space.add(l2, l1)
+        assert l3.strategy is l2.strategy
+
     def test_mul(self):
         l1 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         l2 = l1.mul(2)
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3442,6 +3442,29 @@
 
         a.build_types(f, [str])
 
+    def test_negative_number_find(self):
+        def f(s, e):
+            return "xyz".find("x", s, e)
+
+        a = self.RPythonAnnotator()
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        def f(s, e):
+            return "xyz".rfind("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+
+        def f(s, e):
+            return "xyz".count("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        
+
     def test_setslice(self):
         def f():
             lst = [2, 5, 7]
@@ -4080,7 +4103,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify instances with no common base class" 
+        assert ("RPython cannot unify instances with no common base class"
                 in exc.value.msg)
 
     def test_unionerror_iters(self):
@@ -4096,7 +4119,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify incompatible iterator variants" in 
+        assert ("RPython cannot unify incompatible iterator variants" in
                 exc.value.msg)
 
     def test_variable_getattr(self):
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -10,7 +10,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeFloat, SomeIterator,
     SomePBC, SomeTypedAddressAccess, SomeAddress, SomeType, s_ImpossibleValue,
     s_Bool, s_None, unionof, missing_operation, add_knowntypedata,
-    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString)
+    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper
 from rpython.annotator import builtin
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
@@ -333,12 +333,13 @@
         check_negative_slice(s_start, s_stop)
         lst.listdef.resize()
 
-def check_negative_slice(s_start, s_stop):
+def check_negative_slice(s_start, s_stop, error="slicing"):
     if isinstance(s_start, SomeInteger) and not s_start.nonneg:
-        raise AnnotatorError("slicing: not proven to have non-negative start")
+        raise AnnotatorError("%s: not proven to have non-negative start" %
+                             error)
     if isinstance(s_stop, SomeInteger) and not s_stop.nonneg and \
            getattr(s_stop, 'const', 0) != -1:
-        raise AnnotatorError("slicing: not proven to have non-negative stop")
+        raise AnnotatorError("%s: not proven to have non-negative stop" % error)
 
 
 class __extend__(SomeDict):
@@ -448,12 +449,15 @@
         return s_Bool
 
     def method_find(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "find")
         return SomeInteger()
 
     def method_rfind(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "rfind")
         return SomeInteger()
 
     def method_count(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "count")
         return SomeInteger(nonneg=True)
 
     def method_strip(str, chr):
@@ -520,6 +524,11 @@
     op_contains.can_only_throw = []
 
 
+class __extend__(SomeByteArray):
+    def getslice(ba, s_start, s_stop):
+        check_negative_slice(s_start, s_stop)
+        return SomeByteArray()
+
 class __extend__(SomeUnicodeString):
     def method_encode(uni, s_enc):
         if not s_enc.is_constant():
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -252,6 +252,23 @@
         from rpython.translator.tool.graphpage import try_show
         try_show(self)
 
+    def get_graph(self):
+        import gc
+        pending = [self]   # pending blocks
+        seen = {self: True, None: True}
+        for x in pending:
+            for y in gc.get_referrers(x):
+                if isinstance(y, FunctionGraph):
+                    return y
+                elif isinstance(y, Link):
+                    block = y.prevblock
+                    if block not in seen:
+                        pending.append(block)
+                        seen[block] = True
+                elif isinstance(y, dict):
+                    pending.append(y)   # go back from the dict to the real obj
+        return pending
+
     view = show
 
 
diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py
--- a/rpython/flowspace/specialcase.py
+++ b/rpython/flowspace/specialcase.py
@@ -1,3 +1,4 @@
+import os
 from rpython.flowspace.model import Constant, const
 
 SPECIAL_CASES = {}
@@ -37,6 +38,18 @@
     return space.frame.do_operation('simple_call', const(isinstance),
             w_instance, w_type)
 
+ at register_flow_sc(open)
+def sc_open(space, *args_w):
+    from rpython.rlib.rfile import create_file
+
+    return space.frame.do_operation("simple_call", const(create_file), *args_w)
+
+ at register_flow_sc(os.tmpfile)
+def sc_os_tmpfile(space):
+    from rpython.rlib.rfile import create_temp_rfile
+
+    return space.frame.do_operation("simple_call", const(create_temp_rfile))
+
 # _________________________________________________________________________
 # a simplified version of the basic printing routines, for RPython programs
 class StdOutBuffer:
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1201,6 +1201,7 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+            assert self.card_page_indices > 0
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -1,55 +1,241 @@
 
-""" This file makes open() and friends RPython
+""" This file makes open() and friends RPython. Note that RFile should not
+be used directly and instead it's magically appearing each time you call
+python builtin open()
 """
 
 import os
-from rpython.annotator.model import SomeObject, SomeString, SomeInteger
-from rpython.rtyper.extregistry import ExtRegistryEntry
-from rpython.rtyper.extfunc import register_external
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.tool import rffi_platform as platform
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib import rposix
+from rpython.rlib.rstring import StringBuilder
 
-class SomeFile(SomeObject):
-    def method_write(self, s_arg):
-        assert isinstance(s_arg, SomeString)
+eci = ExternalCompilationInfo(includes=['stdio.h', 'unistd.h', 'sys/types.h'])
 
-    def method_read(self, s_arg=None):
-        if s_arg is not None:
-            assert isinstance(s_arg, SomeInteger)
-        return SomeString(can_be_None=False)
+def llexternal(*args):
+    return rffi.llexternal(*args, compilation_info=eci)
 
-    def method_close(self):
-        pass
+FILE = lltype.Struct('FILE') # opaque type maybe
 
-    def method_seek(self, s_arg, s_whence=None):
-        assert isinstance(s_arg, SomeInteger)
-        if s_whence is not None:
-            assert isinstance(s_whence, SomeInteger)
+class CConfig(object):
+    _compilation_info_ = eci
 
-    def rtyper_makekey(self):
-        return self.__class__,
+    off_t = platform.SimpleType('off_t')
 
-    def rtyper_makerepr(self, rtyper):
-        from rpython.rtyper.lltypesystem.rfile import FileRepr
+CC = platform.configure(CConfig)
+OFF_T = CC['off_t']
 
-        return FileRepr(rtyper)
+c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
+c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
+c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
+                                     lltype.Ptr(FILE)], rffi.SIZE_T)
+c_read = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
+                                   lltype.Ptr(FILE)], rffi.SIZE_T)
+c_feof = llexternal('feof', [lltype.Ptr(FILE)], rffi.INT)
+c_ferror = llexternal('ferror', [lltype.Ptr(FILE)], rffi.INT)
+c_clearerror = llexternal('clearerr', [lltype.Ptr(FILE)], lltype.Void)
+c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT],
+                          rffi.INT)
+c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
+c_fileno = llexternal('fileno', [lltype.Ptr(FILE)], rffi.INT)
+c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], lltype.Signed)
+c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT)
+c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT)
+c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, lltype.Ptr(FILE)],
+                     rffi.CCHARP)
 
-class FileEntry(ExtRegistryEntry):
-    _about_ = open
+BASE_BUF_SIZE = 4096
+BASE_LINE_SIZE = 100
 
-    def compute_result_annotation(self, s_name, s_mode=None):
-        assert isinstance(s_name, SomeString)
-        if s_mode is not None:
-            assert isinstance(s_mode, SomeString)
-        return SomeFile()
+def create_file(filename, mode="r", buffering=-1):
+    assert buffering == -1
+    assert filename is not None
+    assert mode is not None
+    ll_name = rffi.str2charp(filename)
+    try:
+        ll_mode = rffi.str2charp(mode)
+        try:
+            ll_f = c_open(ll_name, ll_mode)
+            if not ll_f:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+        finally:
+            lltype.free(ll_mode, flavor='raw')
+    finally:
+        lltype.free(ll_name, flavor='raw')
+    return RFile(ll_f)
 
-    def specialize_call(self, hop):
-        return hop.r_result.rtype_constructor(hop)
+def create_temp_rfile():
+    res = c_tmpfile()
+    if not res:
+        errno = rposix.get_errno()
+        raise OSError(errno, os.strerror(errno))
+    return RFile(res)
 
-class OSTempfileEntry(ExtRegistryEntry):
-    _about_ = os.tmpfile
+class RFile(object):
+    def __init__(self, ll_file):
+        self.ll_file = ll_file
 
-    def compute_result_annotation(self):
-        return SomeFile()
+    def write(self, value):
+        assert value is not None
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        assert value is not None
+        ll_value = rffi.get_nonmovingbuffer(value)
+        try:
+            # note that since we got a nonmoving buffer, it is either raw
+            # or already cannot move, so the arithmetics below are fine
+            total_bytes = 0
+            ll_current = ll_value
+            while total_bytes < len(value):
+                bytes = c_write(ll_current, 1, len(value) - r_uint(total_bytes),
+                                ll_file)
+                if bytes == 0:
+                    errno = rposix.get_errno()
+                    raise OSError(errno, os.strerror(errno))
+                total_bytes += bytes
+                ll_current = rffi.cast(rffi.CCHARP,
+                                       rffi.cast(lltype.Unsigned, ll_value) +
+                                       total_bytes)
+        finally:
+            rffi.free_nonmovingbuffer(value, ll_value)
 
-    def specialize_call(self, hop):
-        return hop.r_result.rtype_tempfile(hop)
+    def close(self):
+        if self.ll_file:
+            # double close is allowed
+            res = c_close(self.ll_file)
+            self.ll_file = lltype.nullptr(FILE)
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
 
+    def read(self, size=-1):
+        # XXX CPython uses a more delicate logic here
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        if size < 0:
+            # read the entire contents
+            buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
+            try:
+                s = StringBuilder()
+                while True:
+                    returned_size = c_read(buf, 1, BASE_BUF_SIZE, ll_file)
+                    if returned_size == 0:
+                        if c_feof(ll_file):
+                            # ok, finished
+                            return s.build()
+                        errno = c_ferror(ll_file)
+                        c_clearerror(ll_file)
+                        raise OSError(errno, os.strerror(errno))
+                    s.append_charpsize(buf, returned_size)
+            finally:
+                lltype.free(buf, flavor='raw')
+        else:
+            raw_buf, gc_buf = rffi.alloc_buffer(size)
+            try:
+                returned_size = c_read(raw_buf, 1, size, ll_file)
+                if returned_size == 0:
+                    if not c_feof(ll_file):
+                        errno = c_ferror(ll_file)
+                        raise OSError(errno, os.strerror(errno))
+                s = rffi.str_from_buffer(raw_buf, gc_buf, size,
+                                         rffi.cast(lltype.Signed, returned_size))
+            finally:
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+            return s
+
+    def seek(self, pos, whence=0):
+        ll_file = self.ll_file
+        if not ll_file:
+            raise ValueError("I/O operation on closed file")
+        res = c_fseek(ll_file, pos, whence)
+        if res == -1:
+            errno = rposix.get_errno()
+            raise OSError(errno, os.strerror(errno))
+
+    def fileno(self):
+        if self.ll_file:
+            return intmask(c_fileno(self.ll_file))
+        raise ValueError("I/O operation on closed file")
+
+    def tell(self):
+        if self.ll_file:
+            res = intmask(c_ftell(self.ll_file))
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return res
+        raise ValueError("I/O operation on closed file")
+
+    def flush(self):
+        if self.ll_file:
+            res = c_fflush(self.ll_file)
+            if res != 0:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return
+        raise ValueError("I/O operation on closed file")
+
+    def truncate(self, arg=-1):
+        if self.ll_file:
+            if arg == -1:
+                arg = self.tell()
+            res = c_ftruncate(self.fileno(), arg)
+            if res == -1:
+                errno = rposix.get_errno()
+                raise OSError(errno, os.strerror(errno))
+            return
+        raise ValueError("I/O operation on closed file")
+
+    def __del__(self):
+        self.close()
+
+    def _readline1(self, raw_buf):
+        result = c_fgets(raw_buf, BASE_LINE_SIZE, self.ll_file)
+        if not result:
+            if c_feof(self.ll_file):   # ok
+                return 0
+            errno = c_ferror(self.ll_file)
+            raise OSError(errno, os.strerror(errno))
+        #
+        # Assume that fgets() works as documented, and additionally
+        # never writes beyond the final \0, which the CPython
+        # fileobject.c says appears to be the case everywhere.
+        # The only case where the buffer was not big enough is the
+        # case where the buffer is full, ends with \0, and doesn't
+        # end with \n\0.
+        strlen = 0
+        while raw_buf[strlen] != '\0':
+            strlen += 1
+        if (strlen == BASE_LINE_SIZE - 1 and
+              raw_buf[BASE_LINE_SIZE - 2] != '\n'):
+            return -1    # overflow!
+        # common case
+        return strlen
+
+    def readline(self):
+        if self.ll_file:
+            raw_buf, gc_buf = rffi.alloc_buffer(BASE_LINE_SIZE)
+            try:
+                c = self._readline1(raw_buf)
+                if c >= 0:
+                    return rffi.str_from_buffer(raw_buf, gc_buf,
+                                                BASE_LINE_SIZE, c)
+                #
+                # this is the rare case: the line is longer than BASE_LINE_SIZE
+                s = StringBuilder()
+                while True:
+                    s.append_charpsize(raw_buf, BASE_LINE_SIZE - 1)
+                    c = self._readline1(raw_buf)
+                    if c >= 0:
+                        break
+                #
+                s.append_charpsize(raw_buf, c)
+                return s.build()
+            finally:
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+        raise ValueError("I/O operation on closed file")
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -78,3 +78,100 @@
 
         f()
         self.interpret(f, [])
+
+    def test_fileno(self):
+        fname = str(self.tmpdir.join('file_5'))
+
+        def f():
+            f = open(fname, "w")
+            try:
+                return f.fileno()
+            finally:
+                f.close()
+
+        res = self.interpret(f, [])
+        assert res > 2
+
+    def test_tell(self):
+        fname = str(self.tmpdir.join('file_tell'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            try:
+                return f.tell()
+            finally:
+                f.close()
+
+        res = self.interpret(f, [])
+        assert res == 3
+
+    def test_flush(self):
+        fname = str(self.tmpdir.join('file_flush'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            f.flush()
+            f2 = open(fname)
+            assert f2.read() == "xyz"
+            f2.close()
+            f.close()
+
+        self.interpret(f, [])
+
+    def test_truncate(self):
+        fname = str(self.tmpdir.join('file_trunc'))
+
+        def f():
+            f = open(fname, "w")
+            f.write("xyz")
+            f.seek(0)
+            f.truncate(2)
+            f.close()
+            f2 = open(fname)
+            assert f2.read() == "xy"
+            f2.close()
+
+        f()
+        self.interpret(f, [])
+
+
+class TestDirect:
+    def setup_class(cls):
+        cls.tmpdir = udir.join('test_rfile_direct')
+        cls.tmpdir.ensure(dir=True)
+
+    def test_readline(self):
+        fname = str(self.tmpdir.join('file_readline'))
+        j = 0
+        expected = []
+        with open(fname, 'w') as f:
+            for i in range(250):
+                s = ''.join([chr(32+(k&63)) for k in range(j, j + i)])
+                j += 1
+                print >> f, s
+        expected = open(fname).readlines()
+        expected += ['', '']
+        assert len(expected) == 252
+
+        f = rfile.create_file(fname, 'r')
+        for j in range(252):
+            got = f.readline()
+            assert got == expected[j]
+        f.close()
+
+    def test_readline_without_eol_at_the_end(self):
+        fname = str(self.tmpdir.join('file_readline_without_eol_at_the_end'))
+        for n in [1, 10, 97, 98, 99, 100, 101, 102, 103, 150,
+                  196, 197, 198, 199, 200, 201, 202, 203, 204, 250]:
+            s = ''.join([chr(32+(k&63)) for k in range(n)])
+            with open(fname, 'wb') as f:
+                f.write(s)
+
+            f = rfile.create_file(fname, 'r')
+            got = f.readline()
+            assert got == s
+            got = f.readline()
+            assert got == ''
+            f.close()
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -301,6 +301,8 @@
             addr.get_port() == 80):
             found = True
     assert found, lst
+    # The following might fail if the DNS redirects failed requests to a
+    # catch-all address (i.e. opendns).
     e = py.test.raises(GAIError, getaddrinfo, 'www.very-invalidaddress.com', None)
     assert isinstance(e.value.get_msg(), str)
 
diff --git a/rpython/rlib/types.py b/rpython/rlib/types.py
--- a/rpython/rlib/types.py
+++ b/rpython/rlib/types.py
@@ -39,8 +39,12 @@
     return model.SomeUnicodeString(no_nul=True)
 
 
-def str():
-    return model.SomeString()
+def str(can_be_None=False):
+    return model.SomeString(can_be_None=can_be_None)
+
+
+def bytearray():
+    return model.SomeByteArray()
 
 
 def str0():
diff --git a/rpython/rtyper/lltypesystem/rbytearray.py b/rpython/rtyper/lltypesystem/rbytearray.py
--- a/rpython/rtyper/lltypesystem/rbytearray.py
+++ b/rpython/rtyper/lltypesystem/rbytearray.py
@@ -16,6 +16,9 @@
                                                                   lltype.Char,
                                                                   'bytearray_from_str')
 
+def _empty_bytearray():
+    return empty
+
 BYTEARRAY.become(lltype.GcStruct('rpy_bytearray',
                  ('chars', lltype.Array(lltype.Char)), adtmeths={
     'malloc' : lltype.staticAdtMethod(mallocbytearray),
@@ -23,8 +26,11 @@
     'copy_contents_from_str': lltype.staticAdtMethod(
                                          copy_bytearray_contents_from_str),
     'length': rstr.LLHelpers.ll_length,
+    'empty': lltype.staticAdtMethod(_empty_bytearray),
 }))
 
+empty = lltype.malloc(BYTEARRAY, 0, immortal=True)
+
 class LLHelpers(rstr.LLHelpers):
     @classmethod
     def ll_strsetitem(cls, s, i, item):
diff --git a/rpython/rtyper/lltypesystem/rfile.py b/rpython/rtyper/lltypesystem/rfile.py
deleted file mode 100644
--- a/rpython/rtyper/lltypesystem/rfile.py
+++ /dev/null
@@ -1,195 +0,0 @@
-
-import os
-from rpython.rlib import rposix
-from rpython.rlib.rarithmetic import r_uint
-from rpython.annotator import model as annmodel
-from rpython.rtyper.rtyper import Repr
-from rpython.rlib.rstring import StringBuilder
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
-from rpython.rtyper.lltypesystem.rstr import string_repr, STR
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.annlowlevel import hlstr
-from rpython.rtyper.lltypesystem.lloperation import llop
-
-FILE = lltype.Struct('FILE') # opaque type maybe
-FILE_WRAPPER = lltype.GcStruct("FileWrapper", ('file', lltype.Ptr(FILE)))
-
-eci = ExternalCompilationInfo(includes=['stdio.h'])
-
-def llexternal(*args):
-    return rffi.llexternal(*args, compilation_info=eci)
-
-c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
-c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
-c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
-                                     lltype.Ptr(FILE)], rffi.SIZE_T)
-c_read = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
-                                   lltype.Ptr(FILE)], rffi.SIZE_T)
-c_feof = llexternal('feof', [lltype.Ptr(FILE)], rffi.INT)
-c_ferror = llexternal('ferror', [lltype.Ptr(FILE)], rffi.INT)
-c_clearerror = llexternal('clearerr', [lltype.Ptr(FILE)], lltype.Void)
-c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT],
-                          rffi.INT)
-c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
-
-def ll_open(name, mode):
-    file_wrapper = lltype.malloc(FILE_WRAPPER)
-    ll_name = rffi.str2charp(name)
-    ll_mode = rffi.str2charp(mode)
-    try:
-        ll_f = c_open(ll_name, ll_mode)
-        if not ll_f:
-            errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
-        file_wrapper.file = ll_f
-    finally:
-        lltype.free(ll_name, flavor='raw')
-        lltype.free(ll_mode, flavor='raw')
-    return file_wrapper
-
-def ll_tmpfile():
-    file_wrapper = lltype.malloc(FILE_WRAPPER)
-    res = c_tmpfile()
-    if not res:
-        errno = rposix.get_errno()
-        raise OSError(errno, os.strerror(errno))
-    file_wrapper.file = res
-    return file_wrapper
-
-def ll_write(file_wrapper, value):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    value = hlstr(value)
-    assert value is not None
-    ll_value = rffi.get_nonmovingbuffer(value)
-    try:
-        # note that since we got a nonmoving buffer, it is either raw
-        # or already cannot move, so the arithmetics below are fine
-        total_bytes = 0
-        ll_current = ll_value
-        while total_bytes < len(value):
-            bytes = c_write(ll_current, 1, len(value) - r_uint(total_bytes),
-                            ll_file)
-            if bytes == 0:
-                errno = rposix.get_errno()
-                raise OSError(errno, os.strerror(errno))
-            total_bytes += bytes
-            ll_current = rffi.cast(rffi.CCHARP,
-                                   rffi.cast(lltype.Unsigned, ll_value) +
-                                   total_bytes)
-    finally:
-        rffi.free_nonmovingbuffer(value, ll_value)
-
-BASE_BUF_SIZE = 4096
-
-def ll_read(file_wrapper, size):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    if size < 0:
-        # read the entire contents
-        buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw')
-        try:
-            s = StringBuilder()
-            while True:
-                returned_size = c_read(buf, 1, BASE_BUF_SIZE, ll_file)
-                if returned_size == 0:
-                    if c_feof(ll_file):
-                        # ok, finished
-                        return s.build()
-                    errno = c_ferror(ll_file)
-                    c_clearerror(ll_file)
-                    raise OSError(errno, os.strerror(errno))
-                s.append_charpsize(buf, returned_size)
-        finally:
-            lltype.free(buf, flavor='raw')
-    else:
-        raw_buf, gc_buf = rffi.alloc_buffer(size)
-        try:
-            returned_size = c_read(raw_buf, 1, size, ll_file)
-            if returned_size == 0:
-                if not c_feof(ll_file):
-                    errno = c_ferror(ll_file)
-                    raise OSError(errno, os.strerror(errno))
-            s = rffi.str_from_buffer(raw_buf, gc_buf, size,
-                                     rffi.cast(lltype.Signed, returned_size))
-        finally:
-            rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
-        return s
-def ll_seek(file_wrapper, pos, whence):
-    ll_file = file_wrapper.file
-    if not ll_file:
-        raise ValueError("I/O operation on closed file")
-    res = c_fseek(ll_file, pos, whence)
-    if res == -1:
-        errno = rposix.get_errno()
-        raise OSError(errno, os.strerror(errno))
-
-def ll_close(file_wrapper):
-    if file_wrapper.file:
-        # double close is allowed
-        res = c_close(file_wrapper.file)
-        file_wrapper.file = lltype.nullptr(FILE)
-        if res == -1:
-            errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
-
-class FileRepr(Repr):
-    lowleveltype = lltype.Ptr(FILE_WRAPPER)
-
-    def __init__(self, typer):
-        Repr.__init__(self)
-
-    def rtype_constructor(self, hop):
-        repr = hop.rtyper.getrepr(annmodel.SomeString())
-        arg_0 = hop.inputarg(repr, 0)
-        if len(hop.args_v) == 1:
-            arg_1 = hop.inputconst(string_repr, "r")
-        else:
-            arg_1 = hop.inputarg(repr, 1)
-        hop.exception_is_here()
-        open = hop.rtyper.getannmixlevel().delayedfunction(
-            ll_open, [annmodel.SomeString()] * 2,
-            annmodel.SomePtr(self.lowleveltype))
-        v_open = hop.inputconst(lltype.typeOf(open), open)
-        return hop.genop('direct_call', [v_open, arg_0, arg_1],
-                         resulttype=self)
-
-    def rtype_tempfile(self, hop):
-        tmpfile = hop.rtyper.getannmixlevel().delayedfunction(
-            ll_tmpfile, [], annmodel.SomePtr(self.lowleveltype))
-        v_tmpfile = hop.inputconst(lltype.typeOf(tmpfile), tmpfile)
-        hop.exception_is_here()
-        return hop.genop('direct_call', [v_tmpfile], resulttype=self)
-
-
-    def rtype_method_write(self, hop):
-        args_v = hop.inputargs(self, string_repr)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_write, *args_v)
-
-    def rtype_method_close(self, hop):
-        r_self = hop.inputarg(self, 0)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_close, r_self)
-
-    def rtype_method_read(self, hop):
-        r_self = hop.inputarg(self, 0)
-        if len(hop.args_v) != 2:
-            arg_1 = hop.inputconst(lltype.Signed, -1)
-        else:
-            arg_1 = hop.inputarg(lltype.Signed, 1)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_read, r_self, arg_1)
-
-    def rtype_method_seek(self, hop):
-        r_self = hop.inputarg(self, 0)
-        arg_1 = hop.inputarg(lltype.Signed, 1)
-        if len(hop.args_v) != 3:
-            arg_2 = hop.inputconst(lltype.Signed, os.SEEK_SET)
-        else:
-            arg_2 = hop.inputarg(lltype.Signed, 2)
-        hop.exception_is_here()
-        return hop.gendirectcall(ll_seek, r_self, arg_1, arg_2)
-
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -1685,7 +1685,7 @@
         def tmpnam_llimpl():
             return rffi.charp2str(os_tmpnam(lltype.nullptr(rffi.CCHARP.TO)))
 
-        return extdef([], None, llimpl=tmpnam_llimpl,
+        return extdef([], str, llimpl=tmpnam_llimpl,
                       export_name="ll_os.ll_os_tmpnam")
 
 # --------------------------- os.stat & variants ---------------------------
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -187,13 +187,15 @@
         if hop.nb_args > 2:
             v_start = hop.inputarg(Signed, arg=2)
             if not hop.args_s[2].nonneg:
-                raise TyperError("str.find() start must be proven non-negative")
+                raise TyperError("str.%s() start must be proven non-negative"
+                                 % (reverse and 'rfind' or 'find',))
         else:
             v_start = hop.inputconst(Signed, 0)
         if hop.nb_args > 3:
             v_end = hop.inputarg(Signed, arg=3)
             if not hop.args_s[3].nonneg:
-                raise TyperError("str.find() end must be proven non-negative")
+                raise TyperError("str.%s() end must be proven non-negative"
+                                 % (reverse and 'rfind' or 'find',))
         else:
             v_end = hop.gendirectcall(self.ll.ll_strlen, v_str)
         hop.exception_cannot_occur()
diff --git a/rpython/rtyper/test/test_rbytearray.py b/rpython/rtyper/test/test_rbytearray.py
--- a/rpython/rtyper/test/test_rbytearray.py
+++ b/rpython/rtyper/test/test_rbytearray.py
@@ -50,3 +50,10 @@
 
         ll_res = self.interpret(f, [123])
         assert hlstr(ll_res) == "123"
+
+    def test_getslice(self):
+        def f(x):
+            return str(bytearray(str(x))[1:2])
+
+        ll_res = self.interpret(f, [123])
+        assert hlstr(ll_res) == "2"
diff --git a/rpython/rtyper/test/test_rint.py b/rpython/rtyper/test/test_rint.py
--- a/rpython/rtyper/test/test_rint.py
+++ b/rpython/rtyper/test/test_rint.py
@@ -85,6 +85,14 @@
         res = self.ll_to_string(res)
         assert res == '-0x8' + '0' * (len(res)-4)
 
+    def test_hex_of_uint(self):
+        def dummy(i):
+            return hex(r_uint(i))
+
+        res = self.interpret(dummy, [-5])
+        res = self.ll_to_string(res)
+        assert res == '0x' + 'f' * (len(res)-3) + 'b'
+
     def test_oct_of_int(self):
         def dummy(i):
             return oct(i)
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -416,6 +416,14 @@
             res = self.interpret(f, [i])
             assert res == expected
 
+    def test_rfind_error_message(self):
+        const = self.const
+        def f(i):
+            return const("abc").rfind(const(''), i)
+        e = py.test.raises(TyperError, self.interpret, f, [-5])
+        assert str(e.value).startswith(
+            'str.rfind() start must be proven non-negative')
+
     def test_find_char(self):
         const = self.const
         def fn(ch):
@@ -1134,4 +1142,4 @@
         array = lltype.malloc(TP, 12, flavor='raw')
         self.interpret(f, [array, 4])
         assert list(array) == list('abc'*4)
-        lltype.free(array, flavor='raw')
\ No newline at end of file
+        lltype.free(array, flavor='raw')
diff --git a/rpython/translator/c/src/stacklet/switch_arm_gcc.h b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
--- a/rpython/translator/c/src/stacklet/switch_arm_gcc.h
+++ b/rpython/translator/c/src/stacklet/switch_arm_gcc.h
@@ -1,9 +1,8 @@
-#if __ARM_ARCH__ >= 5
-# define call_reg(x) "blx " #x "\n"
-#elif defined (__ARM_ARCH_4T__)
+#if defined(__ARM_ARCH_4__) || defined (__ARM_ARCH_4T__)
 # define call_reg(x) "mov lr, pc ; bx " #x "\n"
 #else
-# define call_reg(x) "mov lr, pc ; mov pc, " #x "\n"
+/* ARM >= 5 */
+# define call_reg(x) "blx " #x "\n"
 #endif
 
 static void __attribute__((optimize("O3"))) *slp_switch(void *(*save_state)(void*, void*),
diff --git a/rpython/translator/tool/graphpage.py b/rpython/translator/tool/graphpage.py
--- a/rpython/translator/tool/graphpage.py
+++ b/rpython/translator/tool/graphpage.py
@@ -200,7 +200,7 @@
                 dotgen.emit_edge(nameof(cdef), nameof(prevcdef), color="red")
             prevcdef = cdef
             cdef = cdef.basedef
-        
+
         self.source = dotgen.generate(target=None)
 
     def followlink(self, name):
@@ -224,7 +224,7 @@
         dotgen.emit('mclimit=15.0')
 
         self.do_compute(dotgen, *args, **kwds)
-        
+
         self.source = dotgen.generate(target=None)
 
         # link the function names to the individual flow graphs
@@ -264,7 +264,7 @@
                 data = self.labelof(classdef, classdef.shortname)
                 dotgen.emit_node(nameof(classdef), label=data, shape="box")
                 dotgen.emit_edge(nameof(classdef.basedef), nameof(classdef))
-             
+
     def labelof(self, obj, objname):
         name = objname
         i = 1
@@ -409,22 +409,11 @@
     elif isinstance(obj, Link):
         try_show(obj.prevblock)
     elif isinstance(obj, Block):
-        import gc
-        pending = [obj]   # pending blocks
-        seen = {obj: True, None: True}
-        for x in pending:
-            for y in gc.get_referrers(x):
-                if isinstance(y, FunctionGraph):
-                    y.show()
-                    return
-                elif isinstance(y, Link):
-                    block = y.prevblock
-                    if block not in seen:
-                        pending.append(block)
-                        seen[block] = True
-                elif isinstance(y, dict):
-                    pending.append(y)   # go back from the dict to the real obj
-        graph = IncompleteGraph(pending)
+        graph = obj.get_graph()
+        if isinstance(graph, FunctionGraph):
+            graph.show()
+            return
+        graph = IncompleteGraph(graph)
         SingleGraphPage(graph).display()
     else:
         raise TypeError("try_show(%r object)" % (type(obj).__name__,))
@@ -449,7 +438,7 @@
                         seen[block] = True
         return pending
     else:
-        raise TypeError("try_get_functiongraph(%r object)" % (type(obj).__name__,))    
+        raise TypeError("try_get_functiongraph(%r object)" % (type(obj).__name__,))
 
 class IncompleteGraph:
     name = '(incomplete graph)'

From noreply at buildbot.pypy.org  Thu Oct 10 01:22:24 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 10 Oct 2013 01:22:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: reapply py3k's behavior
Message-ID: <20131009232224.9F0311C36AD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67272:1a9a32b0a81d
Date: 2013-10-09 16:21 -0700
http://bitbucket.org/pypy/pypy/changeset/1a9a32b0a81d/

Log:	reapply py3k's behavior

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -401,9 +401,10 @@
     try:
         result = _pow(space, x, y)
     except PowDomainError:
-        raise operationerrfmt(space.w_ValueError,
-                             "negative number cannot be raised to a "
-                              "fractional power")
+        # Negative numbers raised to fractional powers become complex
+        return space.pow(space.newcomplex(x, 0.0),
+                         space.newcomplex(y, 0.0),
+                         thirdArg)
     return W_FloatObject(result)
 
 class PowDomainError(ValueError):

From noreply at buildbot.pypy.org  Thu Oct 10 01:28:52 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 10 Oct 2013 01:28:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: whitespace
Message-ID: <20131009232852.D32191C36AD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r67273:bafa606b36e1
Date: 2013-10-09 16:28 -0700
http://bitbucket.org/pypy/pypy/changeset/bafa606b36e1/

Log:	whitespace

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -428,7 +428,7 @@
         result = _pow(space, x, y)
     except PowDomainError:
         raise operationerrfmt(space.w_ValueError,
-                             "negative number cannot be raised to a "
+                              "negative number cannot be raised to a "
                               "fractional power")
     return W_FloatObject(result)
 

From noreply at buildbot.pypy.org  Thu Oct 10 09:32:33 2013
From: noreply at buildbot.pypy.org (dstufft)
Date: Thu, 10 Oct 2013 09:32:33 +0200 (CEST)
Subject: [pypy-commit] cffi default: Fixes #110 - Support different so
	suffixes
Message-ID: <20131010073233.B7EC21C0F38@cobra.cs.uni-duesseldorf.de>

Author: Donald Stufft <donald at stufft.io>
Branch: 
Changeset: r1373:77da9e592d23
Date: 2013-10-06 16:34 -0400
http://bitbucket.org/cffi/cffi/changeset/77da9e592d23/

Log:	Fixes #110 - Support different so suffixes

diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -15,7 +15,7 @@
     def patch_extension_kwds(self, kwds):
         pass
 
-    def find_module(self, module_name, path, so_suffix):
+    def find_module(self, module_name, path, so_suffixes):
         try:
             f, filename, descr = imp.find_module(module_name, path)
         except ImportError:
@@ -25,7 +25,7 @@
         # Note that after a setuptools installation, there are both .py
         # and .so files with the same basename.  The code here relies on
         # imp.find_module() locating the .so in priority.
-        if descr[0] != so_suffix:
+        if descr[0] not in so_suffixes:
             return None
         return filename
 
diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -20,15 +20,15 @@
         # up in kwds['export_symbols'].
         kwds.setdefault('export_symbols', self.export_symbols)
 
-    def find_module(self, module_name, path, so_suffix):
-        basename = module_name + so_suffix
-        if path is None:
-            path = sys.path
-        for dirname in path:
-            filename = os.path.join(dirname, basename)
-            if os.path.isfile(filename):
-                return filename
-        return None
+    def find_module(self, module_name, path, so_suffixes):
+        for so_suffix in so_suffixes:
+            basename = module_name + so_suffix
+            if path is None:
+                path = sys.path
+            for dirname in path:
+                filename = os.path.join(dirname, basename)
+                if os.path.isfile(filename):
+                    return filename
 
     def collect_types(self):
         pass      # not needed in the generic engine
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -31,7 +31,7 @@
             k2 = k2.lstrip('0').rstrip('L')
             modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key,
                                               k1, k2)
-        suffix = _get_so_suffix()
+        suffix = _get_so_suffixes()[0]
         self.tmpdir = tmpdir or _caller_dir_pycache()
         self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
         self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
@@ -103,7 +103,7 @@
             else:
                 path = None
             filename = self._vengine.find_module(self.get_module_name(), path,
-                                                 _get_so_suffix())
+                                                 _get_so_suffixes())
             if filename is None:
                 return
             self.modulefilename = filename
@@ -193,7 +193,7 @@
     if keep_so:
         suffix = '.c'   # only remove .c files
     else:
-        suffix = _get_so_suffix().lower()
+        suffix = _get_so_suffixes()[0].lower()
     for fn in filelist:
         if fn.lower().startswith('_cffi_') and (
                 fn.lower().endswith(suffix) or fn.lower().endswith('.c')):
@@ -213,15 +213,20 @@
         except OSError:
             pass
 
-def _get_so_suffix():
+def _get_so_suffixes():
+    suffixes = []
     for suffix, mode, type in imp.get_suffixes():
         if type == imp.C_EXTENSION:
-            return suffix
-    # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
-    if sys.platform == 'win32':
-        return ".pyd"
-    else:
-        return ".so"
+            suffixes.append(suffix)
+
+    if not suffixes:
+        # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
+        if sys.platform == 'win32':
+            suffixes = [".pyd"]
+        else:
+            suffixes = [".so"]
+
+    return suffixes
 
 def _ensure_dir(filename):
     try:

From noreply at buildbot.pypy.org  Thu Oct 10 09:35:55 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 10 Oct 2013 09:35:55 +0200 (CEST)
Subject: [pypy-commit] cffi default: Fix the test for 77da9e592d23.
Message-ID: <20131010073555.F2F2F1C0F38@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1374:2c78dd9872e7
Date: 2013-10-10 09:34 +0200
http://bitbucket.org/cffi/cffi/changeset/2c78dd9872e7/

Log:	Fix the test for 77da9e592d23.

diff --git a/testing/test_zdistutils.py b/testing/test_zdistutils.py
--- a/testing/test_zdistutils.py
+++ b/testing/test_zdistutils.py
@@ -1,7 +1,7 @@
 import sys, os, imp, math, shutil
 import py
 from cffi import FFI, FFIError
-from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffix
+from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes
 from cffi.ffiplatform import maybe_relative_path
 from testing.udir import udir
 
@@ -249,7 +249,7 @@
         lib = ffi.verify(csrc, force_generic_engine=self.generic,
                          modulename=modname)
         assert lib.test1foo(143) == 80.0
-        suffix = _get_so_suffix()
+        suffix = _get_so_suffixes()[0]
         fn1 = os.path.join(ffi.verifier.tmpdir, modname + '.c')
         fn2 = os.path.join(ffi.verifier.tmpdir, modname + suffix)
         assert ffi.verifier.sourcefilename == fn1

From noreply at buildbot.pypy.org  Thu Oct 10 10:57:41 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 10 Oct 2013 10:57:41 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: rpython fix
Message-ID: <20131010085741.301781C0113@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67274:597b01c88a5a
Date: 2013-10-10 10:57 +0200
http://bitbucket.org/pypy/pypy/changeset/597b01c88a5a/

Log:	rpython fix

diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -69,13 +69,13 @@
         float_strategy = self.space.fromcache(FloatListStrategy)
         #
         if w_ob.strategy is int_stragegy and self.ctitem.is_long():
-            int_list = w_ob.strategy.unerase(w_ob.lstorage)
+            int_list = IntegerListStrategy.unerase(w_ob.lstorage)
             cdata = rffi.cast(rffi.LONGP, cdata)
             copy_list_to_raw_array(int_list, cdata)
             return True
         #
         if w_ob.strategy is float_strategy and self.ctitem.is_double():
-            float_list = w_ob.strategy.unerase(w_ob.lstorage)
+            float_list = FloatListStrategy.unerase(w_ob.lstorage)
             cdata = rffi.cast(rffi.DOUBLEP, cdata)
             copy_list_to_raw_array(float_list, cdata)
             return True

From noreply at buildbot.pypy.org  Thu Oct 10 11:24:46 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 10 Oct 2013 11:24:46 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: introduce
	space.listview_float
Message-ID: <20131010092446.64F5C1C0113@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67275:90ba1db68c0a
Date: 2013-10-10 11:21 +0200
http://bitbucket.org/pypy/pypy/changeset/90ba1db68c0a/

Log:	introduce space.listview_float

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -292,6 +292,11 @@
         """Return the items in the list as unwrapped ints. If the list does not
         use the list strategy, return None."""
         return self.strategy.getitems_int(self)
+
+    def getitems_float(self):
+        """Return the items in the list as unwrapped floats. If the list does not
+        use the list strategy, return None."""
+        return self.strategy.getitems_float(self)
     # ___________________________________________________
 
     def mul(self, times):
@@ -757,6 +762,9 @@
     def getitems_int(self, w_list):
         return None
 
+    def getitems_float(self, w_list):
+        return None
+
     def getstorage_copy(self, w_list):
         raise NotImplementedError
 
@@ -1575,6 +1583,9 @@
         if reverse:
             l.reverse()
 
+    def getitems_float(self, w_list):
+        return self.unerase(w_list.lstorage)
+
 
 class StringListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -472,6 +472,15 @@
             return w_obj.getitems_int()
         return None
 
+    def listview_float(self, w_obj):
+        if type(w_obj) is W_ListObject:
+            return w_obj.getitems_float()
+        # dict and set don't have FloatStrategy, so we can just ignore them
+        # for now
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_float()
+        return None
+
     def view_as_kwargs(self, w_dict):
         if type(w_dict) is W_DictMultiObject:
             return w_dict.view_as_kwargs()
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -645,6 +645,11 @@
         w_l = W_ListObject(space, [space.wrap(1), space.wrap(2), space.wrap(3)])
         assert self.space.listview_int(w_l) == [1, 2, 3]
 
+    def test_listview_float_list(self):
+        space = self.space
+        w_l = W_ListObject(space, [space.wrap(1.1), space.wrap(2.2), space.wrap(3.3)])
+        assert self.space.listview_float(w_l) == [1.1, 2.2, 3.3]
+
 
 class TestW_ListStrategiesDisabled:
     spaceconfig = {"objspace.std.withliststrategies": False}

From noreply at buildbot.pypy.org  Thu Oct 10 11:24:48 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 10 Oct 2013 11:24:48 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: (antocuni,
 cfbolz around): use space.listview_{int,
 float} to get the internal lists of the strategies
Message-ID: <20131010092448.886B51C0113@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67276:a3f89d180a33
Date: 2013-10-10 11:24 +0200
http://bitbucket.org/pypy/pypy/changeset/a3f89d180a33/

Log:	(antocuni, cfbolz around): use space.listview_{int,float} to get the
	internal lists of the strategies

diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -60,22 +60,15 @@
 
     def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
         from rpython.rlib.rarray import copy_list_to_raw_array
-        from pypy.objspace.std.listobject import (W_ListObject,
-                                                  IntegerListStrategy, FloatListStrategy)
-        if not isinstance(w_ob, W_ListObject):
-            return False
+        int_list = self.space.listview_int(w_ob)
+        float_list = self.space.listview_float(w_ob)
         #
-        int_stragegy = self.space.fromcache(IntegerListStrategy)
-        float_strategy = self.space.fromcache(FloatListStrategy)
-        #
-        if w_ob.strategy is int_stragegy and self.ctitem.is_long():
-            int_list = IntegerListStrategy.unerase(w_ob.lstorage)
+        if self.ctitem.is_long() and int_list is not None:
             cdata = rffi.cast(rffi.LONGP, cdata)
             copy_list_to_raw_array(int_list, cdata)
             return True
         #
-        if w_ob.strategy is float_strategy and self.ctitem.is_double():
-            float_list = FloatListStrategy.unerase(w_ob.lstorage)
+        if self.ctitem.is_double() and float_list is not None:
             cdata = rffi.cast(rffi.DOUBLEP, cdata)
             copy_list_to_raw_array(float_list, cdata)
             return True

From noreply at buildbot.pypy.org  Thu Oct 10 11:39:54 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 10 Oct 2013 11:39:54 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: add default implementations
 of these to baseobjspace
Message-ID: <20131010093954.BF6F71C0113@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67277:60e13a44605f
Date: 2013-10-10 11:39 +0200
http://bitbucket.org/pypy/pypy/changeset/60e13a44605f/

Log:	add default implementations of these to baseobjspace

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -895,6 +895,20 @@
         """
         return None
 
+    def listview_int(self, w_list):
+        """ Return a list of unwrapped int out of a list of int. If the
+        argument is not a list or does not contain only int, return None.
+        May return None anyway.
+        """
+        return None
+
+    def listview_float(self, w_list):
+        """ Return a list of unwrapped float out of a list of float. If the
+        argument is not a list or does not contain only float, return None.
+        May return None anyway.
+        """
+        return None
+
     def view_as_kwargs(self, w_dict):
         """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
         strings and one of wrapped values. otherwise return (None, None)

From noreply at buildbot.pypy.org  Thu Oct 10 12:05:35 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 10 Oct 2013 12:05:35 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal,
	arigo) ll_dict_contains
Message-ID: <20131010100535.3E5141C02EA@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67278:9da9581079b8
Date: 2013-10-10 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/9da9581079b8/

Log:	(fijal, arigo) ll_dict_contains

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -376,7 +376,7 @@
     def rtype_contains((r_dict, r_key), hop):
         v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
         hop.exception_is_here()
-        return hop.gendirectcall(ll_contains, v_dict, v_key)
+        return hop.gendirectcall(ll_dict_contains, v_dict, v_key)
 
 class __extend__(pairtype(DictRepr, DictRepr)):
     def convert_from_to((r_dict1, r_dict2), v, llops):
@@ -1076,9 +1076,9 @@
 ll_dict_values = _make_ll_keys_values_items('values')
 ll_dict_items  = _make_ll_keys_values_items('items')
 
-def ll_contains(d, key):
-    i = ll_dict_lookup(d, key, d.keyhash(key))
-    return not i & HIGHEST_BIT
+def ll_dict_contains(d, key):
+    i = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
+    return i != -1
 
 def _ll_getnextitem(dic):
     if dic.num_items == 0:
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -177,6 +177,12 @@
             for k in foreach_index(ll_d):
                 assert k < rdict.VALID_OFFSET
 
+    def test_contains(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        assert rdict.ll_dict_contains(ll_d, llstr("k"))
+        assert not rdict.ll_dict_contains(ll_d, llstr("j"))
 
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:

From noreply at buildbot.pypy.org  Thu Oct 10 12:48:02 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 12:48:02 +0200 (CEST)
Subject: [pypy-commit] pypy cpyext-best_base: document branches to be merged
Message-ID: <20131010104802.A50321C0113@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-best_base
Changeset: r67279:127a235ca3eb
Date: 2013-10-10 12:51 +0300
http://bitbucket.org/pypy/pypy/changeset/127a235ca3eb/

Log:	document branches to be merged

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -52,6 +52,9 @@
 .. branch: ndarray-subtype
 Allow subclassing ndarray, i.e. matrix
 
+.. branch: ndarray-sort
+Implement ndarray in-place sorting (for numeric types, no non-native byte order)
+
 .. branch: pypy-pyarray
 Implement much of numpy's c api in cpyext, allows (slow) access to ndarray
 from c
@@ -87,6 +90,7 @@
 .. branch: no-release-gil
 .. branch: safe-win-mmap
 .. branch: boolean-indexing-cleanup
+.. branch: cpyyest-best_base
 
 .. branch: nobold-backtrace
 Work on improving UnionError messages and stack trace displays.
@@ -103,3 +107,5 @@
 
 .. branch: file-support-in-rpython
 make open() and friends rpython
+
+

From noreply at buildbot.pypy.org  Thu Oct 10 12:48:03 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 12:48:03 +0200 (CEST)
Subject: [pypy-commit] pypy cpyext-best_base: close branch to be merged
Message-ID: <20131010104803.D27E71C0113@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-best_base
Changeset: r67280:89c502c80949
Date: 2013-10-10 12:52 +0300
http://bitbucket.org/pypy/pypy/changeset/89c502c80949/

Log:	close branch to be merged


From noreply at buildbot.pypy.org  Thu Oct 10 12:48:05 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 12:48:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge cpyext-best_base which uses the
 best_base function from pypy.objspace.std.typeobject
Message-ID: <20131010104805.521E41C0113@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67281:289551a68457
Date: 2013-10-10 13:45 +0300
http://bitbucket.org/pypy/pypy/changeset/289551a68457/

Log:	merge cpyext-best_base which uses the best_base function from
	pypy.objspace.std.typeobject

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -52,6 +52,9 @@
 .. branch: ndarray-subtype
 Allow subclassing ndarray, i.e. matrix
 
+.. branch: ndarray-sort
+Implement ndarray in-place sorting (for numeric types, no non-native byte order)
+
 .. branch: pypy-pyarray
 Implement much of numpy's c api in cpyext, allows (slow) access to ndarray
 from c
@@ -87,6 +90,7 @@
 .. branch: no-release-gil
 .. branch: safe-win-mmap
 .. branch: boolean-indexing-cleanup
+.. branch: cpyyest-best_base
 
 .. branch: nobold-backtrace
 Work on improving UnionError messages and stack trace displays.
@@ -103,3 +107,5 @@
 
 .. branch: file-support-in-rpython
 make open() and friends rpython
+
+
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -99,7 +99,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii'])
+                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
@@ -196,7 +196,7 @@
         assert PyUnicode_GetDefaultEncoding() == 'ascii'
 
 class AppTestCpythonExtensionBase(LeakCheckingTest):
-    
+
     def setup_class(cls):
         cls.space.getbuiltinmodule("cpyext")
         from pypy.module.imp.importing import importhook
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -156,7 +156,7 @@
             def __init__(self):
                 self.foobar = 32
                 super(UnicodeSubclass2, self).__init__()
-        
+
         newobj = UnicodeSubclass2()
         assert newobj.get_val() == 42
         assert newobj.foobar == 32
@@ -358,6 +358,13 @@
         assert w_obj is None
         assert api.PyErr_Occurred() is None
 
+    def test_ndarray_ref(self, space, api):
+        w_obj = space.appexec([], """():
+            import numpypy as np
+            return np.int64(2)""")
+        ref = make_ref(space, w_obj)
+        api.Py_DecRef(ref)
+
 class AppTestSlots(AppTestCpythonExtensionBase):
     def test_some_slots(self):
         module = self.import_extension('foo', [
@@ -525,7 +532,7 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
-        
+
     def test_bool(self):
         module = self.import_extension('foo', [
             ("newInt", "METH_VARARGS",
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -4,7 +4,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.annlowlevel import llhelper
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
-from pypy.objspace.std.typeobject import W_TypeObject
+from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
@@ -574,33 +574,7 @@
 def best_base(space, bases_w):
     if not bases_w:
         return None
-
-    w_winner = None
-    w_base = None
-    for w_base_i in bases_w:
-        if isinstance(w_base_i, W_ClassObject):
-            # old-style base
-            continue
-        assert isinstance(w_base_i, W_TypeObject)
-        w_candidate = solid_base(space, w_base_i)
-        if not w_winner:
-            w_winner = w_candidate
-            w_base = w_base_i
-        elif space.abstract_issubclass_w(w_winner, w_candidate):
-            pass
-        elif space.abstract_issubclass_w(w_candidate, w_winner):
-            w_winner = w_candidate
-            w_base = w_base_i
-        else:
-            raise OperationError(
-                space.w_TypeError,
-                space.wrap("multiple bases have instance lay-out conflict"))
-    if w_base is None:
-        raise OperationError(
-            space.w_TypeError,
-                space.wrap("a new-style class can't have only classic bases"))
-
-    return w_base
+    return find_best_base(space, bases_w)
 
 def inherit_slots(space, pto, w_base):
     # XXX missing: nearly everything

From noreply at buildbot.pypy.org  Thu Oct 10 12:48:06 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 12:48:06 +0200 (CEST)
Subject: [pypy-commit] pypy ndarray-sort: close branch to be merged
Message-ID: <20131010104806.7F5D11C0113@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: ndarray-sort
Changeset: r67282:65c514591115
Date: 2013-10-10 13:46 +0300
http://bitbucket.org/pypy/pypy/changeset/65c514591115/

Log:	close branch to be merged


From noreply at buildbot.pypy.org  Thu Oct 10 12:48:08 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 12:48:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge ndarray-sort which implementes
 in-place timsort for numpy numerical ndarrays (not non-native byte order)
Message-ID: <20131010104808.141B11C0113@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67283:000c73872f73
Date: 2013-10-10 13:47 +0300
http://bitbucket.org/pypy/pypy/changeset/000c73872f73/

Log:	merge ndarray-sort which implementes in-place timsort for numpy
	numerical ndarrays (not non-native byte order)

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -356,6 +356,10 @@
         from pypy.module.micronumpy.arrayimpl.sort import argsort_array
         return argsort_array(self, space, w_axis)
 
+    def sort(self, space, w_axis, w_order):
+        from pypy.module.micronumpy.arrayimpl.sort import sort_array
+        return sort_array(self, space, w_axis, w_order)
+
     def base(self):
         return None
 
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -17,7 +17,7 @@
 
 INT_SIZE = rffi.sizeof(lltype.Signed)
 
-def make_sort_function(space, itemtype, comp_type, count=1):
+def make_argsort_function(space, itemtype, comp_type, count=1):
     TP = itemtype.T
     step = rffi.sizeof(TP)
 
@@ -137,8 +137,8 @@
         else:
             shape = arr.get_shape()
             if axis < 0:
-                axis = len(shape) + axis - 1
-            if axis < 0 or axis > len(shape):
+                axis = len(shape) + axis
+            if axis < 0 or axis >= len(shape):
                 raise OperationError(space.w_IndexError, space.wrap(
                                                     "Wrong axis %d" % axis))
             iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
@@ -162,7 +162,7 @@
     return argsort
 
 def argsort_array(arr, space, w_axis):
-    cache = space.fromcache(SortCache) # that populates SortClasses
+    cache = space.fromcache(ArgSortCache) # that populates ArgSortClasses
     itemtype = arr.dtype.itemtype
     for tp in all_types:
         if isinstance(itemtype, tp[0]):
@@ -178,6 +178,166 @@
 all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
 all_types = unrolling_iterable(all_types)
 
+def make_sort_function(space, itemtype, comp_type, count=1):
+    TP = itemtype.T
+    step = rffi.sizeof(TP)
+
+    class Repr(object):
+        def __init__(self, stride_size, size, values, start):
+            self.stride_size = stride_size
+            self.start = start
+            self.size = size
+            self.values = values
+
+        def getitem(self, item):
+            if count < 2:
+                v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                                    + self.start)
+            else:
+                v = []
+                for i in range(count):
+                    _v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                                    + self.start + step * i)
+                    v.append(_v)
+            if comp_type == 'int':
+                v = intmask(v)
+            elif comp_type == 'float':
+                v = float(v)
+            elif comp_type == 'complex':
+                v = [float(v[0]),float(v[1])]
+            else:
+                raise NotImplementedError('cannot reach')
+            return (v)
+
+        def setitem(self, idx, item):
+            if count < 2:
+                raw_storage_setitem(self.values, idx * self.stride_size +
+                                self.start, rffi.cast(TP, item))
+            else:
+                i = 0
+                for val in item:
+                    raw_storage_setitem(self.values, idx * self.stride_size +
+                                self.start + i*step, rffi.cast(TP, val))
+                    i += 1
+
+    class ArgArrayRepWithStorage(Repr):
+        def __init__(self, stride_size, size):
+            start = 0
+            values = alloc_raw_storage(size * stride_size,
+                                            track_allocation=False)
+            Repr.__init__(self, stride_size,
+                          size, values, start)
+
+        def __del__(self):
+            free_raw_storage(self.values, track_allocation=False)
+
+    def arg_getitem(lst, item):
+        return lst.getitem(item)
+
+    def arg_setitem(lst, item, value):
+        lst.setitem(item, value)
+
+    def arg_length(lst):
+        return lst.size
+
+    def arg_getitem_slice(lst, start, stop):
+        retval = ArgArrayRepWithStorage(lst.stride_size, stop-start)
+        for i in range(stop-start):
+            retval.setitem(i, lst.getitem(i+start))
+        return retval
+
+    if count < 2:
+        def arg_lt(a, b):
+            # handles NAN and INF
+            return a < b or b != b and a == a
+    else:
+        def arg_lt(a, b):
+            for i in range(count):
+                if b[i] != b[i] and a[i] == a[i]:
+                    return True
+                elif b[i] == b[i] and a[i] != a[i]:
+                    return False
+            for i in range(count):
+                if a[i] < b[i]:
+                    return True
+                elif a[i] > b[i]:
+                    return False
+            # Does numpy do True?
+            return False
+
+    ArgSort = make_timsort_class(arg_getitem, arg_setitem, arg_length,
+                                 arg_getitem_slice, arg_lt)
+
+    def sort(arr, space, w_axis, itemsize):
+        if w_axis is space.w_None:
+            # note that it's fine to pass None here as we're not going
+            # to pass the result around (None is the link to base in slices)
+            arr = arr.reshape(space, None, [arr.get_size()])
+            axis = 0
+        elif w_axis is None:
+            axis = -1
+        else:
+            axis = space.int_w(w_axis)
+        # create array of indexes
+        if len(arr.get_shape()) == 1:
+            r = Repr(itemsize, arr.get_size(), arr.get_storage(),
+                     arr.start)
+            ArgSort(r).sort()
+        else:
+            shape = arr.get_shape()
+            if axis < 0:
+                axis = len(shape) + axis
+            if axis < 0 or axis >= len(shape):
+                raise OperationError(space.w_IndexError, space.wrap(
+                                                    "Wrong axis %d" % axis))
+            iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
+            iter = AxisIterator(arr, iterable_shape, axis, False)
+            stride_size = arr.strides[axis]
+            axis_size = arr.shape[axis]
+            while not iter.done():
+                r = Repr(stride_size, axis_size, arr.get_storage(), iter.offset)
+                ArgSort(r).sort()
+                iter.next()
+
+    return sort
+
+def sort_array(arr, space, w_axis, w_order):
+    cache = space.fromcache(SortCache) # that populates SortClasses
+    itemtype = arr.dtype.itemtype
+    if not arr.dtype.native:
+        raise OperationError(space.w_NotImplementedError,
+           space.wrap("sorting of non-native btyeorder not supported yet"))
+    for tp in all_types:
+        if isinstance(itemtype, tp[0]):
+            return cache._lookup(tp)(arr, space, w_axis,
+                                     itemtype.get_element_size())
+    # XXX this should probably be changed
+    raise OperationError(space.w_NotImplementedError,
+           space.wrap("sorting of non-numeric types " + \
+                  "'%s' is not implemented" % arr.dtype.get_name(), ))
+
+all_types = (types.all_float_types + types.all_complex_types +
+             types.all_int_types)
+all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
+all_types = unrolling_iterable(all_types)
+
+class ArgSortCache(object):
+    built = False
+
+    def __init__(self, space):
+        if self.built:
+            return
+        self.built = True
+        cache = {}
+        for cls, it in all_types._items:
+            if it == 'complex':
+                cache[cls] = make_argsort_function(space, cls, it, 2)
+            else:
+                cache[cls] = make_argsort_function(space, cls, it)
+        self.cache = cache
+        self._lookup = specialize.memo()(lambda tp : cache[tp[0]])
+
+
 class SortCache(object):
     built = False
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -629,9 +629,13 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "setflags not implemented yet"))
 
-    def descr_sort(self, space, w_axis=-1, w_kind='quicksort', w_order=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "sort not implemented yet"))
+    @unwrap_spec(kind=str)
+    def descr_sort(self, space, w_axis=None, kind='quicksort', w_order=None):
+        # happily ignore the kind
+        # modify the array in-place
+        if self.is_scalar():
+            return
+        return self.implementation.sort(space, w_axis, w_order)
 
     def descr_squeeze(self, space):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -1118,6 +1122,7 @@
     conj = interp2app(W_NDimArray.descr_conj),
 
     argsort  = interp2app(W_NDimArray.descr_argsort),
+    sort  = interp2app(W_NDimArray.descr_sort),
     astype   = interp2app(W_NDimArray.descr_astype),
     base     = GetSetProperty(W_NDimArray.descr_get_base),
     byteswap = interp2app(W_NDimArray.descr_byteswap),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2652,55 +2652,6 @@
         assert array([1, 2, 3], '>i2')[::2].tostring() == '\x00\x01\x00\x03'
         assert array(0, dtype='i2').tostring() == '\x00\x00'
 
-    def test_argsort_dtypes(self):
-        from numpypy import array, arange
-        assert array(2.0).argsort() == 0
-        nnp = self.non_native_prefix
-        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
-                        nnp + 'i2', complex]:
-            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
-            c = a.copy()
-            res = a.argsort()
-            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
-                'a,res,dtype %r,%r,%r' % (a,res,dtype)
-            assert (a == c).all() # not modified
-            a = arange(100)
-            assert (a.argsort() == a).all()
-        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
-
-    def test_argsort_nd(self):
-        from numpypy import array
-        a = array([[4, 2], [1, 3]])
-        assert (a.argsort() == [[1, 0], [0, 1]]).all()
-        a = array(range(10) + range(10) + range(10))
-        b = a.argsort()
-        assert (b[:3] == [0, 10, 20]).all()
-        #trigger timsort 'run' mode which calls arg_getitem_slice
-        a = array(range(100) + range(100) + range(100))
-        b = a.argsort()
-        assert (b[:3] == [0, 100, 200]).all()
-        a = array([[[]]]).reshape(3,4,0)
-        b = a.argsort()
-        assert b.size == 0
-
-    def test_argsort_random(self):
-        from numpypy import array
-        from _random import Random
-        rnd = Random(1)
-        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
-        a.argsort()
-
-    def test_argsort_axis(self):
-        from numpypy import array
-        a = array([[4, 2], [1, 3]])
-        assert (a.argsort(axis=None) == [2, 1, 3, 0]).all()
-        assert (a.argsort(axis=-1) == [[1, 0], [0, 1]]).all()
-        assert (a.argsort(axis=0) == [[1, 0], [0, 1]]).all()
-        assert (a.argsort(axis=1) == [[1, 0], [0, 1]]).all()
-        a = array([[3, 2, 1], [1, 2, 3]])
-        assert (a.argsort(axis=0) == [[1, 0, 0], [0, 1, 1]]).all()
-        assert (a.argsort(axis=1) == [[2, 1, 0], [0, 1, 2]]).all()
-
 
 class AppTestRanges(BaseNumpyAppTest):
     def test_arange(self):
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -0,0 +1,322 @@
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestSupport(BaseNumpyAppTest):
+    def setup_class(cls):
+        import struct
+        BaseNumpyAppTest.setup_class.im_func(cls)
+        cls.w_data = cls.space.wrap(struct.pack('dddd', 1, 2, 3, 4))
+        cls.w_fdata = cls.space.wrap(struct.pack('f', 2.3))
+        cls.w_float16val = cls.space.wrap('\x00E') # 5.0 in float16
+        cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
+        cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
+        cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+
+    def test_argsort_dtypes(self):
+        from numpypy import array, arange
+        assert array(2.0).argsort() == 0
+        nnp = self.non_native_prefix
+        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
+                        nnp + 'i2', complex]:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            c = a.copy()
+            res = a.argsort()
+            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
+                'a,res,dtype %r,%r,%r' % (a,res,dtype)
+            assert (a == c).all() # not modified
+            a = arange(100)
+            assert (a.argsort() == a).all()
+        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
+
+    def test_argsort_nd(self):
+        from numpypy import array
+        a = array([[4, 2], [1, 3]])
+        assert (a.argsort() == [[1, 0], [0, 1]]).all()
+        a = array(range(10) + range(10) + range(10))
+        b = a.argsort()
+        assert (b[:3] == [0, 10, 20]).all()
+        #trigger timsort 'run' mode which calls arg_getitem_slice
+        a = array(range(100) + range(100) + range(100))
+        b = a.argsort()
+        assert (b[:3] == [0, 100, 200]).all()
+        a = array([[[]]]).reshape(3,4,0)
+        b = a.argsort()
+        assert b.size == 0
+
+    def test_argsort_random(self):
+        from numpypy import array
+        from _random import Random
+        rnd = Random(1)
+        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
+        a.argsort()
+
+    def test_argsort_axis(self):
+        from numpypy import array
+        a = array([[4, 2], [1, 3]])
+        assert (a.argsort(axis=None) == [2, 1, 3, 0]).all()
+        assert (a.argsort(axis=-1) == [[1, 0], [0, 1]]).all()
+        assert (a.argsort(axis=0) == [[1, 0], [0, 1]]).all()
+        assert (a.argsort(axis=1) == [[1, 0], [0, 1]]).all()
+        a = array([[3, 2, 1], [1, 2, 3]])
+        assert (a.argsort(axis=0) == [[1, 0, 0], [0, 1, 1]]).all()
+        assert (a.argsort(axis=1) == [[2, 1, 0], [0, 1, 2]]).all()
+
+    def test_sort_dtypes(self):
+        from numpypy import array, arange
+        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
+                        'i2', complex]:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
+            c = a.copy()
+            a.sort()
+            assert (a == b).all(), \
+                'a,orig,dtype %r,%r,%r' % (a,c,dtype)
+        a = arange(100)
+        c = a.copy()
+        a.sort()
+        assert (a == c).all()
+
+    def test_sort_dtypesi_nonnative(self):
+        from numpypy import array
+        nnp = self.non_native_prefix
+        for dtype in [ nnp + 'i2']:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
+            c = a.copy()
+            exc = raises(NotImplementedError, a.sort)
+            assert exc.value[0].find('supported') >= 0
+            #assert (a == b).all(), \
+            #    'a,orig,dtype %r,%r,%r' % (a,c,dtype)
+
+
+# tests from numpy/tests/test_multiarray.py
+    def test_sort_corner_cases(self):
+        # test ordering for floats and complex containing nans. It is only
+        # necessary to check the lessthan comparison, so sorts that
+        # only follow the insertion sort path are sufficient. We only
+        # test doubles and complex doubles as the logic is the same.
+
+        # check doubles
+        from numpypy import array, nan, zeros, complex128, arange
+        from numpy import isnan
+        a = array([nan, 1, 0])
+        b = a.copy()
+        b.sort()
+        assert (isnan(b) == isnan(a[::-1])).all()
+        assert (b[:2] == a[::-1][:2]).all()
+
+        # check complex
+        a = zeros(9, dtype=complex128)
+        a.real += [nan, nan, nan, 1, 0, 1, 1, 0, 0]
+        a.imag += [nan, 1, 0, nan, nan, 1, 0, 1, 0]
+        b = a.copy()
+        b.sort()
+        assert (isnan(b) == isnan(a[::-1])).all()
+        assert (b[:4] == a[::-1][:4]).all()
+
+        # all c scalar sorts use the same code with different types
+        # so it suffices to run a quick check with one type. The number
+        # of sorted items must be greater than ~50 to check the actual
+        # algorithm because quick and merge sort fall over to insertion
+        # sort for small arrays.
+        a = arange(101)
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h'] :
+            msg = "scalar sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+        # test complex sorts. These use the same code as the scalars
+        # but the compare fuction differs.
+        ai = a*1j + 1
+        bi = b*1j + 1
+        for kind in ['q', 'm', 'h'] :
+            msg = "complex sort, real part == 1, kind=%s" % kind
+            c = ai.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+            c = bi.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+        ai = a + 1j
+        bi = b + 1j
+        for kind in ['q', 'm', 'h'] :
+            msg = "complex sort, imag part == 1, kind=%s" % kind
+            c = ai.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+            c = bi.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+
+        # check axis handling. This should be the same for all type
+        # specific sorts, so we only check it for one type and one kind
+        a = array([[3, 2], [1, 0]])
+        b = array([[1, 0], [3, 2]])
+        c = array([[2, 3], [0, 1]])
+        d = a.copy()
+        d.sort(axis=0)
+        assert (d == b).all(), "test sort with axis=0"
+        d = a.copy()
+        d.sort(axis=1)
+        assert (d == c).all(), "test sort with axis=1"
+        d = a.copy()
+        d.sort()
+        assert (d == c).all(), "test sort with default axis"
+
+    def test_sort_corner_cases_string_records(self):
+        skip('not implemented yet')
+        from numpypy import array, dtype
+        # test string sorts.
+        s = 'aaaaaaaa'
+        a = array([s + chr(i) for i in range(101)])
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h'] :
+            msg = "string sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+
+        # test record array sorts.
+        dt =dtype([('f', float), ('i', int)])
+        a = array([(i, i) for i in range(101)], dtype = dt)
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "object sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_unicode(self):
+        from numpypy import array
+        # test unicode sorts.
+        s = 'aaaaaaaa'
+        try:
+            a = array([s + chr(i) for i in range(101)], dtype=unicode)
+            b = a[::-1].copy()
+        except:
+            skip('unicode type not supported yet')
+        for kind in ['q', 'm', 'h'] :
+            msg = "unicode sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_objects(self):
+        # test object array sorts.
+        from numpypy import empty
+        try:
+            a = empty((101,), dtype=object)
+        except:
+            skip('object type not supported yet')
+        a[:] = list(range(101))
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "object sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_datetime(self):
+        from numpypy import arange
+        # test datetime64 sorts.
+        try:
+            a = arange(0, 101, dtype='datetime64[D]')
+        except:
+            skip('datetime type not supported yet')
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "datetime64 sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+        # test timedelta64 sorts.
+        a = arange(0, 101, dtype='timedelta64[D]')
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "timedelta64 sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_order(self):
+        from numpypy import array, zeros
+        from sys import byteorder
+        # Test sorting an array with fields
+        skip('not implemented yet')
+        x1 = array([21, 32, 14])
+        x2 = array(['my', 'first', 'name'])
+        x3=array([3.1, 4.5, 6.2])
+        r=zeros(3, dtype=[('id','i'),('word','S5'),('number','f')])
+        r['id'] = x1
+        r['word'] = x2
+        r['number'] = x3
+
+        r.sort(order=['id'])
+        assert (r['id'] == [14, 21, 32]).all()
+        assert (r['word'] == ['name', 'my', 'first']).all()
+        assert max(abs(r['number'] - [6.2, 3.1, 4.5])) < 1e-6
+
+        r.sort(order=['word'])
+        assert (r['id'] == [32, 21, 14]).all()
+        assert (r['word'] == ['first', 'my', 'name']).all()
+        assert max(abs(r['number'] - [4.5, 3.1, 6.2])) < 1e-6
+
+        r.sort(order=['number'])
+        assert (r['id'] == [21, 32, 14]).all()
+        assert (r['word'] == ['my', 'first', 'name']).all()
+        assert max(abs(r['number'] - [3.1, 4.5, 6.2])) < 1e-6
+
+        if byteorder == 'little':
+            strtype = '>i2'
+        else:
+            strtype = '<i2'
+        mydtype = [('name', 'S5'), ('col2', strtype)]
+        r = array([('a', 1), ('b', 255), ('c', 3), ('d', 258)],
+                     dtype= mydtype)
+        r.sort(order='col2')
+        assert (r['col2'] == [1, 3, 255, 258]).all()
+        assert (r == array([('a', 1), ('c', 3), ('b', 255), ('d', 258)],
+                                 dtype=mydtype)).all()
+
+
+# tests from numpy/tests/test_regression.py
+    def test_sort_bigendian(self):
+        skip('not implemented yet')
+        from numpypy import array, dtype
+        a = array(range(11),dtype='float64')
+        c = a.astype(dtype('<f8'))
+        c.sort()
+        assert max(abs(a-c)) < 1e-32
+
+    def test_string_sort_with_zeros(self):
+        skip('not implemented yet')
+        from numpypy import fromstring
+        """Check sort for strings containing zeros."""
+        x = fromstring("\x00\x02\x00\x01", dtype="S2")
+        y = fromstring("\x00\x01\x00\x02", dtype="S2")
+        x.sort(kind='q')
+        assert (x == y).all()
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1864,7 +1864,6 @@
         return interp_boxes.W_StringBox(arr,  0, arr.dtype)
 
     def fill(self, storage, width, box, start, stop, offset):
-        from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArrayNotOwning
         for i in xrange(start, stop, width):
             self._store(storage, i, offset, box)
 
@@ -1873,6 +1872,13 @@
 class UnicodeType(BaseType, BaseStringType):
     T = lltype.UniChar
 
+    @jit.unroll_safe
+    def coerce(self, space, dtype, w_item):
+        if isinstance(w_item, interp_boxes.W_UnicodeBox):
+            return w_item
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "coerce (probably from set_item) not implemented for unicode type"))
+
 NonNativeUnicodeType = UnicodeType
 
 class VoidType(BaseType, BaseStringType):

From noreply at buildbot.pypy.org  Thu Oct 10 14:44:07 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 10 Oct 2013 14:44:07 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal, arigo) clear
Message-ID: <20131010124407.82CC01C0113@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67284:575b3b774200
Date: 2013-10-10 12:08 +0200
http://bitbucket.org/pypy/pypy/changeset/575b3b774200/

Log:	(fijal, arigo) clear

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -328,7 +328,7 @@
     def rtype_method_clear(self, hop):
         v_dict, = hop.inputargs(self)
         hop.exception_cannot_occur()
-        return hop.gendirectcall(ll_clear, v_dict)
+        return hop.gendirectcall(ll_dict_clear, v_dict)
 
     def rtype_method_popitem(self, hop):
         v_dict, = hop.inputargs(self)
@@ -1007,16 +1007,18 @@
     return d
 ll_copy.oopspec = 'dict.copy(dict)'
 
-def ll_clear(d):
-    if (len(d.entries) == DICT_INITSIZE and
-        d.resize_counter == DICT_INITSIZE * 2):
+def ll_dict_clear(d):
+    if d.num_used_items == 0:
         return
+    DICT = lltype.typeOf(d).TO
     old_entries = d.entries
-    d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_INITSIZE)
+    d.entries = DICT.lookup_family.empty_array
+    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
     d.num_items = 0
+    d.num_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
-    old_entries.delete()
-ll_clear.oopspec = 'dict.clear(d)'
+    # old_entries.delete() XXX
+ll_dict_clear.oopspec = 'dict.clear(d)'
 
 def ll_update(dic1, dic2):
     entries = dic2.entries
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -184,6 +184,15 @@
         assert rdict.ll_dict_contains(ll_d, llstr("k"))
         assert not rdict.ll_dict_contains(ll_d, llstr("j"))
 
+    def test_clear(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        rdict.ll_dict_setitem(ll_d, llstr("j"), 1)
+        rdict.ll_dict_setitem(ll_d, llstr("l"), 1)
+        rdict.ll_dict_clear(ll_d)
+        assert ll_d.num_items == 0
+
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
         ll_dummy_value = llstr("dupa")

From noreply at buildbot.pypy.org  Thu Oct 10 14:44:08 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 10 Oct 2013 14:44:08 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: simplify and support
	convert_const
Message-ID: <20131010124408.C05B71C0113@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67285:3acbd992bfb1
Date: 2013-10-10 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/3acbd992bfb1/

Log:	simplify and support convert_const

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -220,7 +220,6 @@
             get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT, **kwd)
 
     def convert_const(self, dictobj):
-        XXX
         from rpython.rtyper.lltypesystem import llmemory
         # get object from bound dict methods
         #dictobj = getattr(dictobj, '__self__', dictobj)
@@ -250,16 +249,14 @@
                 for dictkeycontainer, dictvalue in dictobj._dict.items():
                     llkey = r_key.convert_const(dictkeycontainer.key)
                     llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_insertclean(l_dict, llkey, llvalue,
-                                        dictkeycontainer.hash)
+                    ll_dict_setitem(l_dict, llkey, llvalue)
                 return l_dict
 
             else:
                 for dictkey, dictvalue in dictobj.items():
                     llkey = r_key.convert_const(dictkey)
                     llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_insertclean(l_dict, llkey, llvalue,
-                                        l_dict.keyhash(llkey))
+                    ll_dict_setitem(l_dict, llkey, llvalue)
                 return l_dict
 
     def rtype_len(self, hop):
@@ -853,15 +850,16 @@
     d.resize_counter = DICT_INITSIZE * 2
     return d
 
-def ll_newdict_size(DICT, length_estimate):
-    xxx
-    length_estimate = (length_estimate // 2) * 3
+def ll_newdict_size(DICT, orig_length_estimate):
+    length_estimate = (orig_length_estimate // 2) * 3
     n = DICT_INITSIZE
     while n < length_estimate:
         n *= 2
     d = DICT.allocate()
-    d.entries = DICT.entries.TO.allocate(n)
+    d.entries = DICT.entries.TO.allocate(orig_length_estimate)
+    ll_malloc_indexes_and_choose_lookup(d, n)
     d.num_items = 0
+    d.num_used_items = 0
     d.resize_counter = n * 2
     return d
 

From noreply at buildbot.pypy.org  Thu Oct 10 14:44:10 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 10 Oct 2013 14:44:10 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: dict.get
Message-ID: <20131010124410.1AB751C0113@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67286:4b309f50a85f
Date: 2013-10-10 12:17 +0200
http://bitbucket.org/pypy/pypy/changeset/4b309f50a85f/

Log:	dict.get

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -274,7 +274,7 @@
         v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
                                                  self.value_repr)
         hop.exception_cannot_occur()
-        v_res = hop.gendirectcall(ll_get, v_dict, v_key, v_default)
+        v_res = hop.gendirectcall(ll_dict_get, v_dict, v_key, v_default)
         return self.recast_value(hop.llops, v_res)
 
     def rtype_method_setdefault(self, hop):
@@ -966,12 +966,12 @@
 # _____________________________________________________________
 # methods
 
-def ll_get(dict, key, default):
-    i = ll_dict_lookup(dict, key, dict.keyhash(key))
-    if not i & HIGHEST_BIT:
-        return ll_get_value(dict, i)
+def ll_dict_get(dict, key, default):
+    index = dict.lookup_function(dict, key, dict.keyhash(key), FLAG_LOOKUP)
+    if index == -1:
+        return default
     else:
-        return default
+        return dict.entries[index].value
 
 def ll_setdefault(dict, key, default):
     hash = dict.keyhash(key)
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -193,6 +193,13 @@
         rdict.ll_dict_clear(ll_d)
         assert ll_d.num_items == 0
 
+    def test_get(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        assert rdict.ll_dict_get(ll_d, llstr("k"), 32) == 1
+        assert rdict.ll_dict_get(ll_d, llstr("j"), 32) == 32
+
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
         ll_dummy_value = llstr("dupa")

From noreply at buildbot.pypy.org  Thu Oct 10 14:44:11 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 10 Oct 2013 14:44:11 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal,
	arigo) dict.setdefault
Message-ID: <20131010124411.3F9121C0113@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67287:86b9d18ca286
Date: 2013-10-10 12:21 +0200
http://bitbucket.org/pypy/pypy/changeset/86b9d18ca286/

Log:	(fijal, arigo) dict.setdefault

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -281,7 +281,7 @@
         v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
                                                  self.value_repr)
         hop.exception_cannot_occur()
-        v_res = hop.gendirectcall(ll_setdefault, v_dict, v_key, v_default)
+        v_res = hop.gendirectcall(ll_dict_setdefault, v_dict, v_key, v_default)
         return self.recast_value(hop.llops, v_res)
 
     def rtype_method_copy(self, hop):
@@ -973,14 +973,14 @@
     else:
         return dict.entries[index].value
 
-def ll_setdefault(dict, key, default):
+def ll_dict_setdefault(dict, key, default):
     hash = dict.keyhash(key)
-    i = ll_dict_lookup(dict, key, hash)
-    if not i & HIGHEST_BIT:
-        return ll_get_value(dict, i)
+    index = dict.lookup_function(dict, key, dict.keyhash(key), FLAG_STORE)
+    if index == -1:
+        _ll_dict_setitem_lookup_done(dict, key, default, hash, -1)
+        return default
     else:
-        _ll_dict_setitem_lookup_done(dict, key, default, hash, i)
-        return default
+        return dict.entries[index].value
 
 def ll_copy(dict):
     DICT = lltype.typeOf(dict).TO
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -200,6 +200,15 @@
         assert rdict.ll_dict_get(ll_d, llstr("k"), 32) == 1
         assert rdict.ll_dict_get(ll_d, llstr("j"), 32) == 32
 
+    def test_setdefault(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        assert rdict.ll_dict_setdefault(ll_d, llstr("j"), 42) == 42
+        assert rdict.ll_dict_getitem(ll_d, llstr("j")) == 42
+        assert rdict.ll_dict_setdefault(ll_d, llstr("k"), 42) == 1
+        assert rdict.ll_dict_getitem(ll_d, llstr("k")) == 1
+        
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
         ll_dummy_value = llstr("dupa")

From noreply at buildbot.pypy.org  Thu Oct 10 16:11:30 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 10 Oct 2013 16:11:30 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: Some random minor renamings
Message-ID: <20131010141130.57C391C02D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67288:db1185cfbee7
Date: 2013-10-10 15:08 +0200
http://bitbucket.org/pypy/pypy/changeset/db1185cfbee7/

Log:	Some random minor renamings

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -102,10 +102,10 @@
     LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY,
                                               lltype.Signed, lltype.Signed],
                                              lltype.Signed))
-    LOOKCLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
-                                                 lltype.Signed,
-                                                 lltype.Signed],
-                                                lltype.Void))
+    STORECLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
+                                                  lltype.Signed,
+                                                  lltype.Signed],
+                                                 lltype.Void))
 
     fields =          [ ("num_items", lltype.Signed),
                         ("num_used_items", lltype.Signed),
@@ -148,10 +148,10 @@
                     ('long', lltype.Unsigned)]:
         if name == 'int' and not IS_64BIT:
             continue
-        lookupfn, lookcleanfn = new_lookup_functions(LOOKUP_FUNC,
-                                                     LOOKCLEAN_FUNC, T=T)
+        lookupfn, storecleanfn = new_lookup_functions(LOOKUP_FUNC,
+                                                      STORECLEAN_FUNC, T=T)
         setattr(family, '%s_lookup_function' % name, lookupfn)
-        setattr(family, '%s_insert_clean_function' % name, lookcleanfn)
+        setattr(family, '%s_insert_clean_function' % name, storecleanfn)
     adtmeths['lookup_family'] = family
 
     DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
@@ -610,24 +610,6 @@
     ll_dict_reindex(d, _ll_len_of_d_indexes(d))
 
 
-def ll_dict_insertclean(d, key, value, hash, lookcleanfn):
-    XXXXXXX
-    # Internal routine used by ll_dict_resize() to insert an item which is
-    # known to be absent from the dict.  This routine also assumes that
-    # the dict contains no deleted entries.  This routine has the advantage
-    # of never calling d.keyhash() and d.keyeq(), so it cannot call back
-    # to user code.  ll_dict_insertclean() doesn't resize the dict, either.
-    index = lookcleanfn(d, hash)
-    ENTRY = lltype.typeOf(d.entries).TO.OF
-    entry = d.entries[index]
-    entry.value = value
-    entry.key = key
-    if hasattr(ENTRY, 'f_hash'):     entry.f_hash = hash
-    if hasattr(ENTRY, 'f_valid'):    entry.f_valid = True
-    d.num_items += 1
-    d.num_used_items += 1
-    d.resize_counter -= 3
-
 def ll_dict_delitem(d, key):
     index = d.lookup_function(d, key, d.keyhash(key), FLAG_DELETE)
     if index == -1:
@@ -707,7 +689,7 @@
 FLAG_DELETE = 2
 FLAG_DELETE_TRY_HARD = 3
 
-def new_lookup_functions(LOOKUP_FUNC, LOOKCLEAN_FUNC, T):
+def new_lookup_functions(LOOKUP_FUNC, STORECLEAN_FUNC, T):
     INDEXES = lltype.Ptr(lltype.GcArray(T))
 
     def ll_kill_something(d):
@@ -833,7 +815,7 @@
         indexes[i] = rffi.cast(T, index + VALID_OFFSET)
 
     return (llhelper(LOOKUP_FUNC, ll_dict_lookup),
-            llhelper(LOOKCLEAN_FUNC, ll_dict_store_clean))
+            llhelper(STORECLEAN_FUNC, ll_dict_store_clean))
 
 # ____________________________________________________________
 #

From noreply at buildbot.pypy.org  Thu Oct 10 16:11:31 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 10 Oct 2013 16:11:31 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: - dict.copy()
Message-ID: <20131010141131.9A8821C03DF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67289:52f6e275c032
Date: 2013-10-10 16:10 +0200
http://bitbucket.org/pypy/pypy/changeset/52f6e275c032/

Log:	- dict.copy()
	- fix a bug in ll_dict_lookup

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -287,7 +287,7 @@
     def rtype_method_copy(self, hop):
         v_dict, = hop.inputargs(self)
         hop.exception_cannot_occur()
-        return hop.gendirectcall(ll_copy, v_dict)
+        return hop.gendirectcall(ll_dict_copy, v_dict)
 
     def rtype_method_update(self, hop):
         v_dic1, v_dic2 = hop.inputargs(self, self)
@@ -770,7 +770,7 @@
                     return ll_kill_something(d)
                 return -1
             elif index >= VALID_OFFSET:
-                checkingkey = entries[index].key
+                checkingkey = entries[index - VALID_OFFSET].key
                 if direct_compare and checkingkey == key:
                     if store_flag == FLAG_DELETE:
                         indexes[i] = rffi.cast(T, DELETED)
@@ -964,28 +964,34 @@
     else:
         return dict.entries[index].value
 
-def ll_copy(dict):
+def ll_dict_copy(dict):
     DICT = lltype.typeOf(dict).TO
-    dictsize = len(dict.entries)
-    d = DICT.allocate()
-    d.entries = DICT.entries.TO.allocate(dictsize)
-    d.num_items = dict.num_items
-    d.resize_counter = dict.resize_counter
-    if hasattr(DICT, 'fnkeyeq'):   d.fnkeyeq   = dict.fnkeyeq
-    if hasattr(DICT, 'fnkeyhash'): d.fnkeyhash = dict.fnkeyhash
+    newdict = DICT.allocate()
+    newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
+
+    newdict.num_items = dict.num_items
+    newdict.num_used_items = dict.num_used_items
+    if hasattr(DICT, 'fnkeyeq'):
+        newdict.fnkeyeq = dict.fnkeyeq
+    if hasattr(DICT, 'fnkeyhash'):
+        newdict.fnkeyhash = dict.fnkeyhash
+
     i = 0
-    while i < dictsize:
-        d_entry = d.entries[i]
+    while i < newdict.num_used_items:
+        d_entry = newdict.entries[i]
         entry = dict.entries[i]
-        ENTRY = lltype.typeOf(d.entries).TO.OF
+        ENTRY = lltype.typeOf(newdict.entries).TO.OF
         d_entry.key = entry.key
-        if hasattr(ENTRY, 'f_valid'):    d_entry.f_valid    = entry.f_valid
-        if hasattr(ENTRY, 'f_everused'): d_entry.f_everused = entry.f_everused
+        if hasattr(ENTRY, 'f_valid'):
+            d_entry.f_valid = entry.f_valid
         d_entry.value = entry.value
-        if hasattr(ENTRY, 'f_hash'):     d_entry.f_hash     = entry.f_hash
+        if hasattr(ENTRY, 'f_hash'):
+            d_entry.f_hash = entry.f_hash
         i += 1
-    return d
-ll_copy.oopspec = 'dict.copy(dict)'
+
+    ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
+    return newdict
+ll_dict_copy.oopspec = 'dict.copy(dict)'
 
 def ll_dict_clear(d):
     if d.num_used_items == 0:
@@ -1029,7 +1035,7 @@
     def ll_kvi(LIST, dic):
         res = LIST.ll_newlist(dic.num_items)
         entries = dic.entries
-        dlen = len(entries)
+        dlen = dic.num_used_items
         items = res.ll_items()
         i = 0
         p = 0
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -65,6 +65,15 @@
         rdict.ll_dict_setitem(ll_d, llstr("abc"), 43)
         assert rdict.ll_dict_getitem(ll_d, lls) == 43
 
+    def test_dict_creation_2(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        llab = llstr("ab")
+        llb = llstr("b")
+        rdict.ll_dict_setitem(ll_d, llab, 1)
+        rdict.ll_dict_setitem(ll_d, llb, 2)
+        assert rdict.ll_dict_getitem(ll_d, llb) == 2
+
     def test_dict_store_get(self):
         DICT = self._get_str_dict()
         ll_d = rdict.ll_newdict(DICT)
@@ -208,7 +217,18 @@
         assert rdict.ll_dict_getitem(ll_d, llstr("j")) == 42
         assert rdict.ll_dict_setdefault(ll_d, llstr("k"), 42) == 1
         assert rdict.ll_dict_getitem(ll_d, llstr("k")) == 1
-        
+
+    def test_copy(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        rdict.ll_dict_setitem(ll_d, llstr("j"), 2)
+        ll_d2 = rdict.ll_dict_copy(ll_d)
+        for ll_d3 in [ll_d, ll_d2]:
+            assert rdict.ll_dict_getitem(ll_d3, llstr("k")) == 1
+            assert rdict.ll_dict_get(ll_d3, llstr("j"), 42) == 2
+            assert rdict.ll_dict_get(ll_d3, llstr("i"), 42) == 42
+
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
         ll_dummy_value = llstr("dupa")

From noreply at buildbot.pypy.org  Thu Oct 10 16:24:17 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 10 Oct 2013 16:24:17 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: general progressiness
Message-ID: <20131010142417.E97831C02D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67290:25503fc2783e
Date: 2013-10-10 16:23 +0200
http://bitbucket.org/pypy/pypy/changeset/25503fc2783e/

Log:	general progressiness

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -292,7 +292,7 @@
     def rtype_method_update(self, hop):
         v_dic1, v_dic2 = hop.inputargs(self, self)
         hop.exception_cannot_occur()
-        return hop.gendirectcall(ll_update, v_dic1, v_dic2)
+        return hop.gendirectcall(ll_dict_update, v_dic1, v_dic2)
 
     def _rtype_method_kvi(self, hop, ll_func):
         v_dic, = hop.inputargs(self)
@@ -957,7 +957,7 @@
 
 def ll_dict_setdefault(dict, key, default):
     hash = dict.keyhash(key)
-    index = dict.lookup_function(dict, key, dict.keyhash(key), FLAG_STORE)
+    index = dict.lookup_function(dict, key, hash, FLAG_STORE)
     if index == -1:
         _ll_dict_setitem_lookup_done(dict, key, default, hash, -1)
         return default
@@ -1006,19 +1006,19 @@
     # old_entries.delete() XXX
 ll_dict_clear.oopspec = 'dict.clear(d)'
 
-def ll_update(dic1, dic2):
-    entries = dic2.entries
-    d2len = len(entries)
+def ll_dict_update(dic1, dic2):
     i = 0
-    while i < d2len:
+    while i < dic2.num_used_items:
+        entries = dic2.entries
         if entries.valid(i):
             entry = entries[i]
             hash = entries.hash(i)
             key = entry.key
-            j = ll_dict_lookup(dic1, key, hash)
-            _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j)
+            value = entry.value
+            index = dic1.lookup_function(dic1, key, hash, FLAG_STORE)
+            _ll_dict_setitem_lookup_done(dic1, key, value, hash, index)
         i += 1
-ll_update.oopspec = 'dict.update(dic1, dic2)'
+ll_dict_update.oopspec = 'dict.update(dic1, dic2)'
 
 # this is an implementation of keys(), values() and items()
 # in a single function.
@@ -1081,7 +1081,7 @@
         i -= 1
 
     key = entries[i].key
-    index = dic.lookup_function(dic, key, dic.keyhash(key),
+    index = dic.lookup_function(dic, key, entries.hash(i),
                                 FLAG_DELETE_TRY_HARD)
     # if the lookup function returned me a random strange thing,
     # don't care about deleting the item
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -229,6 +229,18 @@
             assert rdict.ll_dict_get(ll_d3, llstr("j"), 42) == 2
             assert rdict.ll_dict_get(ll_d3, llstr("i"), 42) == 42
 
+    def test_update(self):
+        DICT = self._get_str_dict()
+        ll_d1 = rdict.ll_newdict(DICT)
+        ll_d2 = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d1, llstr("k"), 5)
+        rdict.ll_dict_setitem(ll_d1, llstr("j"), 6)
+        rdict.ll_dict_setitem(ll_d2, llstr("i"), 7)
+        rdict.ll_dict_setitem(ll_d2, llstr("k"), 8)
+        rdict.ll_dict_update(ll_d1, ll_d2)
+        for key, value in [("k", 8), ("i", 7), ("j", 6)]:
+            assert rdict.ll_dict_getitem(ll_d1, llstr(key)) == value
+
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
         ll_dummy_value = llstr("dupa")

From noreply at buildbot.pypy.org  Thu Oct 10 16:25:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 10 Oct 2013 16:25:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: Probably fixes a rare bug involving
 dict.update() and objects with
Message-ID: <20131010142502.637521C02D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67291:47b3936d1f4a
Date: 2013-10-10 16:24 +0200
http://bitbucket.org/pypy/pypy/changeset/47b3936d1f4a/

Log:	Probably fixes a rare bug involving dict.update() and objects with
	custom __eq__() that hack at the dict

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -820,8 +820,9 @@
             entry = entries[i]
             hash = entries.hash(i)
             key = entry.key
+            value = entry.value
             j = ll_dict_lookup(dic1, key, hash)
-            _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j)
+            _ll_dict_setitem_lookup_done(dic1, key, value, hash, j)
         i += 1
 ll_update.oopspec = 'dict.update(dic1, dic2)'
 

From noreply at buildbot.pypy.org  Thu Oct 10 16:26:40 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 10 Oct 2013 16:26:40 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: add
 space.unpackiterable_{int, float}: they are similar to listview_*,
 with the difference that you are free to modify the returned list.
Message-ID: <20131010142640.6EE7E1C02D9@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67292:4af5ccf25009
Date: 2013-10-10 15:16 +0200
http://bitbucket.org/pypy/pypy/changeset/4af5ccf25009/

Log:	add space.unpackiterable_{int,float}: they are similar to
	listview_*, with the difference that you are free to modify the
	returned list.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -239,6 +239,18 @@
         # _____ this code is here to support testing only _____
         return self
 
+    def unpackiterable_int(self, space):
+        lst = space.listview_int(self)
+        if lst:
+            return lst[:]
+        return None
+
+    def unpackiterable_float(self, space):
+        lst = space.listview_float(self)
+        if lst:
+            return lst[:]
+        return None
+
 
 class W_InterpIterable(W_Root):
     def __init__(self, space, w_iterable):
@@ -838,6 +850,22 @@
         return self._unpackiterable_known_length_jitlook(w_iterator,
                                                          expected_length)
 
+
+    def unpackiterable_int(self, w_obj):
+        """
+        Return a RPython list of unwrapped ints out of w_obj. The list is
+        guaranteed to be acopy of the actual data contained in w_obj, so you
+        can freely modify it. It might return None if not supported.
+        """
+        return w_obj.unpackiterable_int(self)
+
+    def unpackiterable_float(self, w_obj):
+        """
+        Same as unpackiterable_int, but for floats.
+        """
+        return w_obj.unpackiterable_float(self)
+
+
     def length_hint(self, w_obj, default):
         """Return the length of an object, consulting its __length_hint__
         method if necessary.
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -650,6 +650,15 @@
         w_l = W_ListObject(space, [space.wrap(1.1), space.wrap(2.2), space.wrap(3.3)])
         assert self.space.listview_float(w_l) == [1.1, 2.2, 3.3]
 
+    def test_unpackiterable_int_list(self):
+        space = self.space
+        w_l = W_ListObject(space, [space.wrap(1), space.wrap(2), space.wrap(3)])
+        list_orig = self.space.listview_int(w_l)
+        list_copy = self.space.unpackiterable_int(w_l)
+        assert list_orig == list_copy == [1, 2, 3]
+        list_copy[0] = 42
+        assert list_orig == [1, 2, 3]
+
 
 class TestW_ListStrategiesDisabled:
     spaceconfig = {"objspace.std.withliststrategies": False}

From noreply at buildbot.pypy.org  Thu Oct 10 16:26:41 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 10 Oct 2013 16:26:41 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: add a fast path to convert
 a cffi long[] array to a list
Message-ID: <20131010142641.9864A1C02D9@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67293:b25eb074c18b
Date: 2013-10-10 15:55 +0200
http://bitbucket.org/pypy/pypy/changeset/b25eb074c18b/

Log:	add a fast path to convert a cffi long[] array to a list

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -282,6 +282,9 @@
     def iter(self):
         return self.ctype.iter(self)
 
+    def unpackiterable_int(self, space):
+        return self.ctype.unpackiterable_int(self)
+
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
         misc.write_raw_signed_data(self._cdata, source, self.ctype.size)
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -105,6 +105,16 @@
     def iter(self, cdata):
         return W_CDataIter(self.space, self.ctitem, cdata)
 
+    def unpackiterable_int(self, cdata):
+        from rpython.rlib.rarray import populate_list_from_raw_array
+        if self.ctitem.is_long():
+            res = []
+            buf = rffi.cast(rffi.LONGP, cdata._cdata)
+            length = cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        return None
+
     def get_vararg_type(self):
         return self.ctptr
 
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -163,6 +163,9 @@
                               "cdata '%s' does not support iteration",
                               self.name)
 
+    def unpackiterable_int(self, cdata):
+        return None
+
     def get_vararg_type(self):
         return self
 
diff --git a/pypy/module/_cffi_backend/test/test_extra.py b/pypy/module/_cffi_backend/test/test_extra.py
--- a/pypy/module/_cffi_backend/test/test_extra.py
+++ b/pypy/module/_cffi_backend/test/test_extra.py
@@ -2,7 +2,7 @@
 from pypy.module._cffi_backend import misc
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
 
-class AppTestFastPath(object):
+class AppTest_fast_path_from_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
 
     def setup_method(self, meth):
@@ -34,3 +34,40 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
+
+class AppTest_fast_path_to_list(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def setup_method(self, meth):
+        from pypy.interpreter import gateway
+        from rpython.rlib import rarray
+        #
+        self.count = 0
+        def get_count(*args):
+            return self.space.wrap(self.count)
+        self.w_get_count = self.space.wrap(gateway.interp2app(get_count))
+        #
+        original = rarray.populate_list_from_raw_array
+        def populate_list_from_raw_array(*args):
+            self.count += 1
+            return original(*args)
+        self._original = original
+        rarray.populate_list_from_raw_array = populate_list_from_raw_array
+
+
+    def teardown_method(self, meth):
+        from rpython.rlib import rarray
+        rarray.populate_list_from_raw_array = self._original
+
+    def test_list_int(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, 3)
+        buf = _cffi_backend.newp(LONG_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        assert self.get_count() == 1
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -950,6 +950,8 @@
             return
 
         intlist = space.listview_int(w_iterable)
+        if intlist is None:
+            intlist = space.unpackiterable_int(w_iterable)
         if intlist is not None:
             w_list.strategy = strategy = space.fromcache(IntegerListStrategy)
             # need to copy because intlist can share with w_iterable

From noreply at buildbot.pypy.org  Thu Oct 10 16:26:42 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 10 Oct 2013 16:26:42 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: add a fast-path also for
 floats. Also, remove the unnecessary copy in case of int lists,
 because now this is already guaranteed by unpackiterable_int
Message-ID: <20131010142642.B41511C02D9@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67294:474e7f3e7d2d
Date: 2013-10-10 16:25 +0200
http://bitbucket.org/pypy/pypy/changeset/474e7f3e7d2d/

Log:	add a fast-path also for floats. Also, remove the unnecessary copy
	in case of int lists, because now this is already guaranteed by
	unpackiterable_int

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -283,7 +283,10 @@
         return self.ctype.iter(self)
 
     def unpackiterable_int(self, space):
-        return self.ctype.unpackiterable_int(self)
+        return self.ctype.aslist_int(self)
+
+    def unpackiterable_float(self, space):
+        return self.ctype.aslist_float(self)
 
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -105,7 +105,7 @@
     def iter(self, cdata):
         return W_CDataIter(self.space, self.ctitem, cdata)
 
-    def unpackiterable_int(self, cdata):
+    def aslist_int(self, cdata):
         from rpython.rlib.rarray import populate_list_from_raw_array
         if self.ctitem.is_long():
             res = []
@@ -115,6 +115,16 @@
             return res
         return None
 
+    def aslist_float(self, cdata):
+        from rpython.rlib.rarray import populate_list_from_raw_array
+        if self.ctitem.is_double():
+            res = []
+            buf = rffi.cast(rffi.DOUBLEP, cdata._cdata)
+            length = cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        return None
+
     def get_vararg_type(self):
         return self.ctptr
 
diff --git a/pypy/module/_cffi_backend/test/test_extra.py b/pypy/module/_cffi_backend/test/test_extra.py
--- a/pypy/module/_cffi_backend/test/test_extra.py
+++ b/pypy/module/_cffi_backend/test/test_extra.py
@@ -71,3 +71,16 @@
         lst = list(buf)
         assert lst == [1, 2, 3]
         assert self.get_count() == 1
+
+    def test_list_float(self):
+        import _cffi_backend
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        P_DOUBLE = _cffi_backend.new_pointer_type(DOUBLE)
+        DOUBLE_ARRAY = _cffi_backend.new_array_type(P_DOUBLE, 3)
+        buf = _cffi_backend.newp(DOUBLE_ARRAY)
+        buf[0] = 1.1
+        buf[1] = 2.2
+        buf[2] = 3.3
+        lst = list(buf)
+        assert lst == [1.1, 2.2, 3.3]
+        assert self.get_count() == 1
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -949,13 +949,16 @@
             w_list.__init__(space, w_iterable.getitems_copy())
             return
 
-        intlist = space.listview_int(w_iterable)
-        if intlist is None:
-            intlist = space.unpackiterable_int(w_iterable)
+        intlist = space.unpackiterable_int(w_iterable)
         if intlist is not None:
             w_list.strategy = strategy = space.fromcache(IntegerListStrategy)
-            # need to copy because intlist can share with w_iterable
-            w_list.lstorage = strategy.erase(intlist[:])
+            w_list.lstorage = strategy.erase(intlist)
+            return
+
+        floatlist = space.unpackiterable_float(w_iterable)
+        if floatlist is not None:
+            w_list.strategy = strategy = space.fromcache(FloatListStrategy)
+            w_list.lstorage = strategy.erase(floatlist)
             return
 
         strlist = space.listview_str(w_iterable)

From noreply at buildbot.pypy.org  Thu Oct 10 17:05:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 10 Oct 2013 17:05:23 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: Fix
Message-ID: <20131010150523.CD4401C11BF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67295:2d1aff779d33
Date: 2013-10-10 16:31 +0200
http://bitbucket.org/pypy/pypy/changeset/2d1aff779d33/

Log:	Fix

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -207,7 +207,9 @@
             assert not self.force_non_null # XXX kill the flag
             kwd = {}
             if self.custom_eq_hash:
-                kwd['get_custom_eq_hash'] = self.custom_eq_hash
+                self.r_rdict_eqfn, self.r_rdict_hashfn = (
+                    self._custom_eq_hash_repr())
+                kwd['get_custom_eq_hash'] = self._custom_eq_hash_repr
             else:
                 kwd['ll_hash_function'] = self.key_repr.get_ll_hash_function()
                 kwd['ll_eq_function'] = self.key_repr.get_ll_eq_function()

From noreply at buildbot.pypy.org  Thu Oct 10 17:05:25 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 10 Oct 2013 17:05:25 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: (fijal, arigo)
Message-ID: <20131010150525.1389F1C11BF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67296:879f9a355048
Date: 2013-10-10 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/879f9a355048/

Log:	(fijal, arigo)

	dict.pop()

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -334,15 +334,15 @@
         r_tuple = hop.r_result
         cTUPLE = hop.inputconst(lltype.Void, r_tuple.lowleveltype)
         hop.exception_is_here()
-        return hop.gendirectcall(ll_popitem, cTUPLE, v_dict)
+        return hop.gendirectcall(ll_dict_popitem, cTUPLE, v_dict)
 
     def rtype_method_pop(self, hop):
         if hop.nb_args == 2:
             v_args = hop.inputargs(self, self.key_repr)
-            target = ll_pop
+            target = ll_dict_pop
         elif hop.nb_args == 3:
             v_args = hop.inputargs(self, self.key_repr, self.value_repr)
-            target = ll_pop_default
+            target = ll_dict_pop_default
         hop.exception_is_here()
         v_res = hop.gendirectcall(target, *v_args)
         return self.recast_value(hop.llops, v_res)
@@ -1093,7 +1093,7 @@
         assert index != -1
     return index
 
-def ll_popitem(ELEM, dic):
+def ll_dict_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
     r = lltype.malloc(ELEM.TO)
@@ -1102,17 +1102,18 @@
     _ll_dict_del(dic, r_uint(i))
     return r
 
-def ll_pop(dic, key):
-    i = ll_dict_lookup(dic, key, dic.keyhash(key))
-    if not i & HIGHEST_BIT:
-        value = ll_get_value(dic, r_uint(i))
-        _ll_dict_del(dic, r_uint(i))
-        return value
-    else:
+def ll_dict_pop(dic, key):
+    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    if index == -1:
         raise KeyError
+    value = dic.entries[index].value
+    _ll_dict_del(dic, index)
+    return value
 
-def ll_pop_default(dic, key, dfl):
-    try:
-        return ll_pop(dic, key)
-    except KeyError:
+def ll_dict_pop_default(dic, key, dfl):
+    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    if index == -1:
         return dfl
+    value = dic.entries[index].value
+    _ll_dict_del(dic, index)
+    return value
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -162,13 +162,13 @@
         rdict.ll_dict_setitem(ll_d, llstr("j"), 2)
         TUP = lltype.Ptr(lltype.GcStruct('x', ('item0', lltype.Ptr(rstr.STR)),
                                               ('item1', lltype.Signed)))
-        ll_elem = rdict.ll_popitem(TUP, ll_d)
+        ll_elem = rdict.ll_dict_popitem(TUP, ll_d)
         assert hlstr(ll_elem.item0) == "j"
         assert ll_elem.item1 == 2
-        ll_elem = rdict.ll_popitem(TUP, ll_d)
+        ll_elem = rdict.ll_dict_popitem(TUP, ll_d)
         assert hlstr(ll_elem.item0) == "k"
         assert ll_elem.item1 == 1
-        py.test.raises(KeyError, rdict.ll_popitem, TUP, ll_d)
+        py.test.raises(KeyError, rdict.ll_dict_popitem, TUP, ll_d)
 
     def test_direct_enter_and_del(self):
         def eq(a, b):
@@ -241,6 +241,26 @@
         for key, value in [("k", 8), ("i", 7), ("j", 6)]:
             assert rdict.ll_dict_getitem(ll_d1, llstr(key)) == value
 
+    def test_pop(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("k"), 5)
+        rdict.ll_dict_setitem(ll_d, llstr("j"), 6)
+        assert rdict.ll_dict_pop(ll_d, llstr("k")) == 5
+        assert rdict.ll_dict_pop(ll_d, llstr("j")) == 6
+        py.test.raises(KeyError, rdict.ll_dict_pop, ll_d, llstr("k"))
+        py.test.raises(KeyError, rdict.ll_dict_pop, ll_d, llstr("j"))
+
+    def test_pop_default(self):
+        DICT = self._get_str_dict()
+        ll_d = rdict.ll_newdict(DICT)
+        rdict.ll_dict_setitem(ll_d, llstr("k"), 5)
+        rdict.ll_dict_setitem(ll_d, llstr("j"), 6)
+        assert rdict.ll_dict_pop_default(ll_d, llstr("k"), 42) == 5
+        assert rdict.ll_dict_pop_default(ll_d, llstr("j"), 41) == 6
+        assert rdict.ll_dict_pop_default(ll_d, llstr("k"), 40) == 40
+        assert rdict.ll_dict_pop_default(ll_d, llstr("j"), 39) == 39
+
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
         ll_dummy_value = llstr("dupa")

From noreply at buildbot.pypy.org  Thu Oct 10 17:05:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 10 Oct 2013 17:05:26 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: Some more tests
	kind-of-passing
Message-ID: <20131010150526.31DB21C11BF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67297:330b97b84a06
Date: 2013-10-10 16:47 +0200
http://bitbucket.org/pypy/pypy/changeset/330b97b84a06/

Log:	Some more tests kind-of-passing

diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1071,9 +1071,9 @@
                     del d[chr(ord('A') - i)]
             return d
         res = self.interpret(func, [0])
-        assert len(res.entries) > rdict.DICT_INITSIZE
+        assert rdict._ll_len_of_d_indexes(res) > rdict.DICT_INITSIZE
         res = self.interpret(func, [1])
-        assert len(res.entries) == rdict.DICT_INITSIZE
+        assert rdict._ll_len_of_d_indexes(res) == rdict.DICT_INITSIZE
 
     def test_dict_valid_resize(self):
         # see if we find our keys after resize
@@ -1378,7 +1378,7 @@
                        DictValue(None, annmodel.SomeString(value_can_be_none)))
         dictrepr.setup()
         print dictrepr.lowleveltype
-        for key, value in dictrepr.DICTENTRY._adtmeths.items():
+        for key, value in dictrepr.DICT.entries.TO._adtmeths.items():
             print '    %s = %s' % (key, value)
         l_dict = rdict.ll_newdict(dictrepr.DICT)
         referencetable = [None] * 400

From noreply at buildbot.pypy.org  Thu Oct 10 17:05:27 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 10 Oct 2013 17:05:27 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: Re-add temporarily the old
 rdict.py, used with rlib/_rweak*dict and with
Message-ID: <20131010150527.586801C11BF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67298:db8a6dd0791a
Date: 2013-10-10 16:54 +0200
http://bitbucket.org/pypy/pypy/changeset/db8a6dd0791a/

Log:	Re-add temporarily the old rdict.py, used with rlib/_rweak*dict and
	with memory/lldict.

diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -497,9 +497,9 @@
     _ll_0_newdict.need_result_type = True
 
     _ll_2_dict_delitem = ll_rdict.ll_dict_delitem
-    _ll_1_dict_copy = ll_rdict.ll_copy
-    _ll_1_dict_clear = ll_rdict.ll_clear
-    _ll_2_dict_update = ll_rdict.ll_update
+    _ll_1_dict_copy = ll_rdict.ll_dict_copy
+    _ll_1_dict_clear = ll_rdict.ll_dict_clear
+    _ll_2_dict_update = ll_rdict.ll_dict_update
 
     # ---------- dict keys(), values(), items(), iter ----------
 
diff --git a/rpython/memory/lldict.py b/rpython/memory/lldict.py
--- a/rpython/memory/lldict.py
+++ b/rpython/memory/lldict.py
@@ -1,5 +1,5 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
-from rpython.rtyper.lltypesystem import rdict
+from rpython.rtyper.lltypesystem import rdict_old as rdict
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.memory.support import mangle_hash
 
diff --git a/rpython/rlib/_rweakkeydict.py b/rpython/rlib/_rweakkeydict.py
--- a/rpython/rlib/_rweakkeydict.py
+++ b/rpython/rlib/_rweakkeydict.py
@@ -1,5 +1,6 @@
 from rpython.flowspace.model import Constant
-from rpython.rtyper.lltypesystem import lltype, llmemory, rclass, rdict
+from rpython.rtyper.lltypesystem import lltype, llmemory, rclass
+from rpython.rtyper.lltypesystem import rdict_old as rdict
 from rpython.rtyper.lltypesystem.llmemory import weakref_create, weakref_deref
 from rpython.rtyper.rclass import getinstancerepr
 from rpython.rtyper.rmodel import Repr
diff --git a/rpython/rlib/_rweakvaldict.py b/rpython/rlib/_rweakvaldict.py
--- a/rpython/rlib/_rweakvaldict.py
+++ b/rpython/rlib/_rweakvaldict.py
@@ -1,5 +1,6 @@
 from rpython.flowspace.model import Constant
-from rpython.rtyper.lltypesystem import lltype, llmemory, rclass, rdict
+from rpython.rtyper.lltypesystem import lltype, llmemory, rclass
+from rpython.rtyper.lltypesystem import rdict_old as rdict
 from rpython.rtyper.lltypesystem.llmemory import weakref_create, weakref_deref
 from rpython.rtyper.rclass import getinstancerepr
 from rpython.rtyper.rmodel import Repr
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -503,7 +503,7 @@
 
 # It may be safe to look inside always, it has a few branches though, and their
 # frequencies needs to be investigated.
-#@jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
+ at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     ENTRY = lltype.typeOf(d.entries).TO.OF
     if i >= 0:
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict_old.py
copy from rpython/rtyper/lltypesystem/rdict.py
copy to rpython/rtyper/lltypesystem/rdict_old.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict_old.py
@@ -1,168 +1,43 @@
-import sys
 from rpython.tool.pairtype import pairtype
 from rpython.flowspace.model import Constant
 from rpython.rtyper.rdict import AbstractDictRepr, AbstractDictIteratorRepr
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.rlib import objectmodel, jit, rgc
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib import objectmodel, jit
 from rpython.rlib.debug import ll_assert
-from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT
 from rpython.rtyper import rmodel
 from rpython.rtyper.error import TyperError
-from rpython.rtyper.annlowlevel import llhelper
 
 
+HIGHEST_BIT = r_uint(intmask(1 << (LONG_BIT - 1)))
+MASK = r_uint(intmask(HIGHEST_BIT - 1))
+
 # ____________________________________________________________
 #
 #  generic implementation of RPython dictionary, with parametric DICTKEY and
-#  DICTVALUE types. The basic implementation is a sparse array of indexes
-#  plus a dense array of structs that contain keys and values. struct looks
-#  like that:
+#  DICTVALUE types.
 #
+#  XXX for immutable dicts, the array should be inlined and
+#      resize_counter and everused are not needed.
 #
 #    struct dictentry {
 #        DICTKEY key;
+#        bool f_valid;      # (optional) the entry is filled
+#        bool f_everused;   # (optional) the entry is or has ever been filled
 #        DICTVALUE value;
-#        long f_hash;        # (optional) key hash, if hard to recompute
-#        bool f_valid;      # (optional) the entry is filled
+#        int f_hash;        # (optional) key hash, if hard to recompute
 #    }
 #
 #    struct dicttable {
 #        int num_items;
-#        int num_used_items;
 #        int resize_counter;
-#        {byte, short, int, long} *indexes;
-#        dictentry *entries;
-#        lookup_function; # one of the four possible functions for different
-#                         # size dicts
+#        Array *entries;
 #        (Function DICTKEY, DICTKEY -> bool) *fnkeyeq;
 #        (Function DICTKEY -> int) *fnkeyhash;
 #    }
 #
 #
 
-def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
-                ll_fasthash_function=None, ll_hash_function=None,
-                ll_eq_function=None, method_cache={},
-                dummykeyobj=None, dummyvalueobj=None):
-    # get the actual DICT type. if DICT is None, it's created, otherwise
-    # forward reference is becoming DICT
-    if DICT is None:
-        DICT = lltype.GcForwardReference()
-    # compute the shape of the DICTENTRY structure
-    entryfields = []
-    entrymeths = {
-        'allocate': lltype.typeMethod(_ll_malloc_entries),
-        'delete': _ll_free_entries,
-        'must_clear_key':   (isinstance(DICTKEY, lltype.Ptr)
-                             and DICTKEY._needsgc()),
-        'must_clear_value': (isinstance(DICTVALUE, lltype.Ptr)
-                             and DICTVALUE._needsgc()),
-        }
-
-    # * the key
-    entryfields.append(("key", DICTKEY))
-
-    # * the state of the entry - trying to encode it as dummy objects
-    if dummykeyobj:
-        # all the state can be encoded in the key
-        entrymeths['dummy_obj'] = dummykeyobj
-        entrymeths['valid'] = ll_valid_from_key
-        entrymeths['mark_deleted'] = ll_mark_deleted_in_key
-        # the key is overwritten by 'dummy' when the entry is deleted
-        entrymeths['must_clear_key'] = False
-
-    elif dummyvalueobj:
-        # all the state can be encoded in the value
-        entrymeths['dummy_obj'] = dummyvalueobj
-        entrymeths['valid'] = ll_valid_from_value
-        entrymeths['mark_deleted'] = ll_mark_deleted_in_value
-        # value is overwritten by 'dummy' when entry is deleted
-        entrymeths['must_clear_value'] = False
-
-    else:
-        # we need a flag to know if the entry was ever used
-        entryfields.append(("f_valid", lltype.Bool))
-        entrymeths['valid'] = ll_valid_from_flag
-        entrymeths['mark_deleted'] = ll_mark_deleted_in_flag
-
-    # * the value
-    entryfields.append(("value", DICTVALUE))
-
-    if ll_fasthash_function is None:
-        entryfields.append(("f_hash", lltype.Signed))
-        entrymeths['hash'] = ll_hash_from_cache
-    else:
-        entrymeths['hash'] = ll_hash_recomputed
-        entrymeths['fasthashfn'] = ll_fasthash_function
-
-    # Build the lltype data structures
-    DICTENTRY = lltype.Struct("dictentry", *entryfields)
-    DICTENTRYARRAY = lltype.GcArray(DICTENTRY,
-                                    adtmeths=entrymeths)
-    LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY,
-                                              lltype.Signed, lltype.Signed],
-                                             lltype.Signed))
-    STORECLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
-                                                  lltype.Signed,
-                                                  lltype.Signed],
-                                                 lltype.Void))
-
-    fields =          [ ("num_items", lltype.Signed),
-                        ("num_used_items", lltype.Signed),
-                        ("resize_counter", lltype.Signed),
-                        ("indexes", llmemory.GCREF),
-                        ("lookup_function", LOOKUP_FUNC),
-                        ("entries", lltype.Ptr(DICTENTRYARRAY)) ]
-    if get_custom_eq_hash is not None:
-        r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash()
-        fields.extend([ ("fnkeyeq", r_rdict_eqfn.lowleveltype),
-                        ("fnkeyhash", r_rdict_hashfn.lowleveltype) ])
-        adtmeths = {
-            'keyhash':        ll_keyhash_custom,
-            'keyeq':          ll_keyeq_custom,
-            'r_rdict_eqfn':   r_rdict_eqfn,
-            'r_rdict_hashfn': r_rdict_hashfn,
-            'paranoia':       True,
-            }
-    else:
-        # figure out which functions must be used to hash and compare
-        ll_keyhash = ll_hash_function
-        ll_keyeq = ll_eq_function
-        ll_keyhash = lltype.staticAdtMethod(ll_keyhash)
-        if ll_keyeq is not None:
-            ll_keyeq = lltype.staticAdtMethod(ll_keyeq)
-        adtmeths = {
-            'keyhash':  ll_keyhash,
-            'keyeq':    ll_keyeq,
-            'paranoia': False,
-            }
-    adtmeths['KEY']   = DICTKEY
-    adtmeths['VALUE'] = DICTVALUE
-    adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
-
-    family = LookupFamily()
-    family.empty_array = DICTENTRYARRAY.allocate(0)
-    for name, T in [('byte', rffi.UCHAR),
-                    ('short', rffi.USHORT),
-                    ('int', rffi.UINT),
-                    ('long', lltype.Unsigned)]:
-        if name == 'int' and not IS_64BIT:
-            continue
-        lookupfn, storecleanfn = new_lookup_functions(LOOKUP_FUNC,
-                                                      STORECLEAN_FUNC, T=T)
-        setattr(family, '%s_lookup_function' % name, lookupfn)
-        setattr(family, '%s_insert_clean_function' % name, storecleanfn)
-    adtmeths['lookup_family'] = family
-
-    DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
-                                *fields))
-    return DICT
-
-class LookupFamily:
-    def _freeze_(self):
-        return True
-
-
 class DictRepr(AbstractDictRepr):
 
     def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
@@ -198,28 +73,138 @@
         if 'value_repr' not in self.__dict__:
             self.external_value_repr, self.value_repr = self.pickrepr(self._value_repr_computer())
         if isinstance(self.DICT, lltype.GcForwardReference):
-            DICTKEY = self.key_repr.lowleveltype
-            DICTVALUE = self.value_repr.lowleveltype
-            # * we need an explicit flag if the key and the value is not
-            #   able to store dummy values
+            self.DICTKEY = self.key_repr.lowleveltype
+            self.DICTVALUE = self.value_repr.lowleveltype
+
+            # compute the shape of the DICTENTRY structure
+            entryfields = []
+            entrymeths = {
+                'allocate': lltype.typeMethod(_ll_malloc_entries),
+                'delete': _ll_free_entries,
+                'must_clear_key':   (isinstance(self.DICTKEY, lltype.Ptr)
+                                     and self.DICTKEY._needsgc()),
+                'must_clear_value': (isinstance(self.DICTVALUE, lltype.Ptr)
+                                     and self.DICTVALUE._needsgc()),
+                }
+
+            # * the key
+            entryfields.append(("key", self.DICTKEY))
+
+            # * if NULL is not a valid ll value for the key or the value
+            #   field of the entry, it can be used as a marker for
+            #   never-used entries.  Otherwise, we need an explicit flag.
             s_key   = self.dictkey.s_value
             s_value = self.dictvalue.s_value
-            assert not self.force_non_null # XXX kill the flag
-            kwd = {}
+            nullkeymarker = not self.key_repr.can_ll_be_null(s_key)
+            nullvaluemarker = not self.value_repr.can_ll_be_null(s_value)
+            if self.force_non_null:
+                if not nullkeymarker:
+                    rmodel.warning("%s can be null, but forcing non-null in dict key" % s_key)
+                    nullkeymarker = True
+                if not nullvaluemarker:
+                    rmodel.warning("%s can be null, but forcing non-null in dict value" % s_value)
+                    nullvaluemarker = True
+            dummykeyobj = self.key_repr.get_ll_dummyval_obj(self.rtyper,
+                                                            s_key)
+            dummyvalueobj = self.value_repr.get_ll_dummyval_obj(self.rtyper,
+                                                                s_value)
+
+            # * the state of the entry - trying to encode it as dummy objects
+            if nullkeymarker and dummykeyobj:
+                # all the state can be encoded in the key
+                entrymeths['everused'] = ll_everused_from_key
+                entrymeths['dummy_obj'] = dummykeyobj
+                entrymeths['valid'] = ll_valid_from_key
+                entrymeths['mark_deleted'] = ll_mark_deleted_in_key
+                # the key is overwritten by 'dummy' when the entry is deleted
+                entrymeths['must_clear_key'] = False
+
+            elif nullvaluemarker and dummyvalueobj:
+                # all the state can be encoded in the value
+                entrymeths['everused'] = ll_everused_from_value
+                entrymeths['dummy_obj'] = dummyvalueobj
+                entrymeths['valid'] = ll_valid_from_value
+                entrymeths['mark_deleted'] = ll_mark_deleted_in_value
+                # value is overwritten by 'dummy' when entry is deleted
+                entrymeths['must_clear_value'] = False
+
+            else:
+                # we need a flag to know if the entry was ever used
+                # (we cannot use a NULL as a marker for this, because
+                # the key and value will be reset to NULL to clear their
+                # reference)
+                entryfields.append(("f_everused", lltype.Bool))
+                entrymeths['everused'] = ll_everused_from_flag
+
+                # can we still rely on a dummy obj to mark deleted entries?
+                if dummykeyobj:
+                    entrymeths['dummy_obj'] = dummykeyobj
+                    entrymeths['valid'] = ll_valid_from_key
+                    entrymeths['mark_deleted'] = ll_mark_deleted_in_key
+                    # key is overwritten by 'dummy' when entry is deleted
+                    entrymeths['must_clear_key'] = False
+                elif dummyvalueobj:
+                    entrymeths['dummy_obj'] = dummyvalueobj
+                    entrymeths['valid'] = ll_valid_from_value
+                    entrymeths['mark_deleted'] = ll_mark_deleted_in_value
+                    # value is overwritten by 'dummy' when entry is deleted
+                    entrymeths['must_clear_value'] = False
+                else:
+                    entryfields.append(("f_valid", lltype.Bool))
+                    entrymeths['valid'] = ll_valid_from_flag
+                    entrymeths['mark_deleted'] = ll_mark_deleted_in_flag
+
+            # * the value
+            entryfields.append(("value", self.DICTVALUE))
+
+            # * the hash, if needed
             if self.custom_eq_hash:
-                self.r_rdict_eqfn, self.r_rdict_hashfn = (
-                    self._custom_eq_hash_repr())
-                kwd['get_custom_eq_hash'] = self._custom_eq_hash_repr
+                fasthashfn = None
             else:
-                kwd['ll_hash_function'] = self.key_repr.get_ll_hash_function()
-                kwd['ll_eq_function'] = self.key_repr.get_ll_eq_function()
-                kwd['ll_fasthash_function'] = self.key_repr.get_ll_fasthash_function()
-            kwd['dummykeyobj'] = self.key_repr.get_ll_dummyval_obj(self.rtyper,
-                                                                   s_key)
-            kwd['dummyvalueobj'] = self.value_repr.get_ll_dummyval_obj(
-                self.rtyper, s_value)
+                fasthashfn = self.key_repr.get_ll_fasthash_function()
+            if fasthashfn is None:
+                entryfields.append(("f_hash", lltype.Signed))
+                entrymeths['hash'] = ll_hash_from_cache
+            else:
+                entrymeths['hash'] = ll_hash_recomputed
+                entrymeths['fasthashfn'] = fasthashfn
 
-            get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT, **kwd)
+            # Build the lltype data structures
+            self.DICTENTRY = lltype.Struct("dictentry", *entryfields)
+            self.DICTENTRYARRAY = lltype.GcArray(self.DICTENTRY,
+                                                 adtmeths=entrymeths)
+            fields =          [ ("num_items", lltype.Signed),
+                                ("resize_counter", lltype.Signed),
+                                ("entries", lltype.Ptr(self.DICTENTRYARRAY)) ]
+            if self.custom_eq_hash:
+                self.r_rdict_eqfn, self.r_rdict_hashfn = self._custom_eq_hash_repr()
+                fields.extend([ ("fnkeyeq", self.r_rdict_eqfn.lowleveltype),
+                                ("fnkeyhash", self.r_rdict_hashfn.lowleveltype) ])
+                adtmeths = {
+                    'keyhash':        ll_keyhash_custom,
+                    'keyeq':          ll_keyeq_custom,
+                    'r_rdict_eqfn':   self.r_rdict_eqfn,
+                    'r_rdict_hashfn': self.r_rdict_hashfn,
+                    'paranoia':       True,
+                    }
+            else:
+                # figure out which functions must be used to hash and compare
+                ll_keyhash = self.key_repr.get_ll_hash_function()
+                ll_keyeq = self.key_repr.get_ll_eq_function()  # can be None
+                ll_keyhash = lltype.staticAdtMethod(ll_keyhash)
+                if ll_keyeq is not None:
+                    ll_keyeq = lltype.staticAdtMethod(ll_keyeq)
+                adtmeths = {
+                    'keyhash':  ll_keyhash,
+                    'keyeq':    ll_keyeq,
+                    'paranoia': False,
+                    }
+            adtmeths['KEY']   = self.DICTKEY
+            adtmeths['VALUE'] = self.DICTVALUE
+            adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
+            self.DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
+                                             *fields))
+
 
     def convert_const(self, dictobj):
         from rpython.rtyper.lltypesystem import llmemory
@@ -251,14 +236,16 @@
                 for dictkeycontainer, dictvalue in dictobj._dict.items():
                     llkey = r_key.convert_const(dictkeycontainer.key)
                     llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_setitem(l_dict, llkey, llvalue)
+                    ll_dict_insertclean(l_dict, llkey, llvalue,
+                                        dictkeycontainer.hash)
                 return l_dict
 
             else:
                 for dictkey, dictvalue in dictobj.items():
                     llkey = r_key.convert_const(dictkey)
                     llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_setitem(l_dict, llkey, llvalue)
+                    ll_dict_insertclean(l_dict, llkey, llvalue,
+                                        l_dict.keyhash(llkey))
                 return l_dict
 
     def rtype_len(self, hop):
@@ -276,25 +263,25 @@
         v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
                                                  self.value_repr)
         hop.exception_cannot_occur()
-        v_res = hop.gendirectcall(ll_dict_get, v_dict, v_key, v_default)
+        v_res = hop.gendirectcall(ll_get, v_dict, v_key, v_default)
         return self.recast_value(hop.llops, v_res)
 
     def rtype_method_setdefault(self, hop):
         v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
                                                  self.value_repr)
         hop.exception_cannot_occur()
-        v_res = hop.gendirectcall(ll_dict_setdefault, v_dict, v_key, v_default)
+        v_res = hop.gendirectcall(ll_setdefault, v_dict, v_key, v_default)
         return self.recast_value(hop.llops, v_res)
 
     def rtype_method_copy(self, hop):
         v_dict, = hop.inputargs(self)
         hop.exception_cannot_occur()
-        return hop.gendirectcall(ll_dict_copy, v_dict)
+        return hop.gendirectcall(ll_copy, v_dict)
 
     def rtype_method_update(self, hop):
         v_dic1, v_dic2 = hop.inputargs(self, self)
         hop.exception_cannot_occur()
-        return hop.gendirectcall(ll_dict_update, v_dic1, v_dic2)
+        return hop.gendirectcall(ll_update, v_dic1, v_dic2)
 
     def _rtype_method_kvi(self, hop, ll_func):
         v_dic, = hop.inputargs(self)
@@ -327,22 +314,22 @@
     def rtype_method_clear(self, hop):
         v_dict, = hop.inputargs(self)
         hop.exception_cannot_occur()
-        return hop.gendirectcall(ll_dict_clear, v_dict)
+        return hop.gendirectcall(ll_clear, v_dict)
 
     def rtype_method_popitem(self, hop):
         v_dict, = hop.inputargs(self)
         r_tuple = hop.r_result
         cTUPLE = hop.inputconst(lltype.Void, r_tuple.lowleveltype)
         hop.exception_is_here()
-        return hop.gendirectcall(ll_dict_popitem, cTUPLE, v_dict)
+        return hop.gendirectcall(ll_popitem, cTUPLE, v_dict)
 
     def rtype_method_pop(self, hop):
         if hop.nb_args == 2:
             v_args = hop.inputargs(self, self.key_repr)
-            target = ll_dict_pop
+            target = ll_pop
         elif hop.nb_args == 3:
             v_args = hop.inputargs(self, self.key_repr, self.value_repr)
-            target = ll_dict_pop_default
+            target = ll_pop_default
         hop.exception_is_here()
         v_res = hop.gendirectcall(target, *v_args)
         return self.recast_value(hop.llops, v_res)
@@ -375,7 +362,7 @@
     def rtype_contains((r_dict, r_key), hop):
         v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
         hop.exception_is_here()
-        return hop.gendirectcall(ll_dict_contains, v_dict, v_key)
+        return hop.gendirectcall(ll_contains, v_dict, v_key)
 
 class __extend__(pairtype(DictRepr, DictRepr)):
     def convert_from_to((r_dict1, r_dict2), v, llops):
@@ -397,71 +384,36 @@
 #  be direct_call'ed from rtyped flow graphs, which means that they will
 #  get flowed and annotated, mostly with SomePtr.
 
-DICTINDEX_LONG = lltype.Ptr(lltype.GcArray(lltype.Unsigned))
-DICTINDEX_INT = lltype.Ptr(lltype.GcArray(rffi.UINT))
-DICTINDEX_SHORT = lltype.Ptr(lltype.GcArray(rffi.USHORT))
-DICTINDEX_BYTE = lltype.Ptr(lltype.GcArray(rffi.UCHAR))
+def ll_everused_from_flag(entries, i):
+    return entries[i].f_everused
 
-IS_64BIT = sys.maxint != 2 ** 31 - 1
+def ll_everused_from_key(entries, i):
+    return bool(entries[i].key)
 
-def ll_malloc_indexes_and_choose_lookup(d, n):
-    DICT = lltype.typeOf(d).TO
-    if n <= 256:
-        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
-                                           lltype.malloc(DICTINDEX_BYTE.TO, n,
-                                                         zero=True))
-        d.lookup_function = DICT.lookup_family.byte_lookup_function
-    elif n <= 65536:
-        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
-                                           lltype.malloc(DICTINDEX_SHORT.TO, n,
-                                                         zero=True))
-        d.lookup_function = DICT.lookup_family.short_lookup_function
-    elif IS_64BIT and n <= 2 ** 32:
-        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
-                                           lltype.malloc(DICTINDEX_INT.TO, n,
-                                                         zero=True))
-        d.lookup_function = DICT.lookup_family.int_lookup_function
-    else:
-        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
-                                           lltype.malloc(DICTINDEX_LONG.TO, n,
-                                                         zero=True))
-        d.lookup_function = DICT.lookup_family.long_lookup_function
-ll_malloc_indexes_and_choose_lookup._always_inline_ = True
-
-def ll_pick_insert_clean_function(d):
-    DICT = lltype.typeOf(d).TO
-    if d.lookup_function == DICT.lookup_family.byte_lookup_function:
-        return DICT.lookup_family.byte_insert_clean_function
-    if d.lookup_function == DICT.lookup_family.short_lookup_function:
-        return DICT.lookup_family.short_insert_clean_function
-    if IS_64BIT:
-        if d.lookup_function == DICT.lookup_family.int_lookup_function:
-            return DICT.lookup_family.int_insert_clean_function
-    if d.lookup_function == DICT.lookup_family.long_lookup_function:
-        return DICT.lookup_family.long_insert_clean_function
-    assert False
+def ll_everused_from_value(entries, i):
+    return bool(entries[i].value)
 
 def ll_valid_from_flag(entries, i):
     return entries[i].f_valid
 
+def ll_mark_deleted_in_flag(entries, i):
+    entries[i].f_valid = False
+
 def ll_valid_from_key(entries, i):
     ENTRIES = lltype.typeOf(entries).TO
     dummy = ENTRIES.dummy_obj.ll_dummy_value
-    return entries[i].key != dummy
-
-def ll_valid_from_value(entries, i):
-    ENTRIES = lltype.typeOf(entries).TO
-    dummy = ENTRIES.dummy_obj.ll_dummy_value
-    return entries[i].value != dummy
-
-def ll_mark_deleted_in_flag(entries, i):
-    entries[i].f_valid = False
+    return entries.everused(i) and entries[i].key != dummy
 
 def ll_mark_deleted_in_key(entries, i):
     ENTRIES = lltype.typeOf(entries).TO
     dummy = ENTRIES.dummy_obj.ll_dummy_value
     entries[i].key = dummy
 
+def ll_valid_from_value(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    return entries.everused(i) and entries[i].value != dummy
+
 def ll_mark_deleted_in_value(entries, i):
     ENTRIES = lltype.typeOf(entries).TO
     dummy = ENTRIES.dummy_obj.ll_dummy_value
@@ -474,6 +426,9 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
+def ll_get_value(d, i):
+    return d.entries[i].value
+
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
     return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
@@ -490,142 +445,79 @@
     return bool(d) and d.num_items != 0
 
 def ll_dict_getitem(d, key):
-    index = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
-    if index != -1:
-        return d.entries[index].value
+    i = ll_dict_lookup(d, key, d.keyhash(key))
+    if not i & HIGHEST_BIT:
+        return ll_get_value(d, i)
     else:
         raise KeyError
 
 def ll_dict_setitem(d, key, value):
     hash = d.keyhash(key)
-    index = d.lookup_function(d, key, hash, FLAG_STORE)
-    return _ll_dict_setitem_lookup_done(d, key, value, hash, index)
+    i = ll_dict_lookup(d, key, hash)
+    return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
 # It may be safe to look inside always, it has a few branches though, and their
 # frequencies needs to be investigated.
-#@jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
+ at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
+    valid = (i & HIGHEST_BIT) == 0
+    i = i & MASK
     ENTRY = lltype.typeOf(d.entries).TO.OF
-    if i >= 0:
-        entry = d.entries[i]
-        entry.value = value
-    else:
-        if len(d.entries) == d.num_used_items:
-            if ll_dict_grow(d):
-                insertcleanfn = ll_pick_insert_clean_function(d)
-                insertcleanfn(d, hash, d.num_used_items)
-        entry = d.entries[d.num_used_items]
-        entry.key = key
-        entry.value = value
-        if hasattr(ENTRY, 'f_hash'):
-            entry.f_hash = hash
-        if hasattr(ENTRY, 'f_valid'):
-            entry.f_valid = True
-        d.num_used_items += 1
-        d.num_items += 1
+    entry = d.entries[i]
+    if not d.entries.everused(i):
+        # a new entry that was never used before
+        ll_assert(not valid, "valid but not everused")
         rc = d.resize_counter - 3
-        if rc <= 0:
+        if rc <= 0:       # if needed, resize the dict -- before the insertion
             ll_dict_resize(d)
+            i = ll_dict_lookup_clean(d, hash)  # then redo the lookup for 'key'
+            entry = d.entries[i]
             rc = d.resize_counter - 3
             ll_assert(rc > 0, "ll_dict_resize failed?")
         d.resize_counter = rc
+        if hasattr(ENTRY, 'f_everused'): entry.f_everused = True
+        entry.value = value
+    else:
+        # override an existing or deleted entry
+        entry.value = value
+        if valid:
+            return
+    entry.key = key
+    if hasattr(ENTRY, 'f_hash'):  entry.f_hash = hash
+    if hasattr(ENTRY, 'f_valid'): entry.f_valid = True
+    d.num_items += 1
 
-def _ll_len_of_d_indexes(d):
-    # xxx Haaaack: returns len(d.indexes).  Works independently of
-    # the exact type pointed to by d, using a forced cast...
-    return len(rffi.cast(DICTINDEX_BYTE, d.indexes))
-
-def _overallocate_entries_len(baselen):
-    # This over-allocates proportional to the list size, making room
-    # for additional growth.  The over-allocation is mild, but is
-    # enough to give linear-time amortized behavior over a long
-    # sequence of appends() in the presence of a poorly-performing
-    # system malloc().
-    # The growth pattern is:  0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
-    newsize = baselen + 1
-    if newsize < 9:
-        some = 3
-    else:
-        some = 6
-    some += newsize >> 3
-    return newsize + some
-
-def ll_dict_grow(d):
-    if d.num_items < d.num_used_items // 4:
-        ll_dict_remove_deleted_items(d)
-        return True
-
-    new_allocated = _overallocate_entries_len(len(d.entries))
-
-    # Detect an obscure case where the indexes numeric type is too
-    # small to store all the entry indexes
-    if (max(128, _ll_len_of_d_indexes(d)) - new_allocated
-                   < MIN_INDEXES_MINUS_ENTRIES):
-        ll_dict_remove_deleted_items(d)
-        return True
-
-    newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
-    #
-    # XXX we should do this with rgc.ll_arraycopy()!!
-    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
-    i = 0
-    while i < len(d.entries):
-        src = d.entries[i]
-        dst = newitems[i]
-        dst.key = src.key
-        dst.value = src.value
-        if hasattr(ENTRY, 'f_hash'):
-            dst.f_hash = src.f_hash
-        if hasattr(ENTRY, 'f_valid'):
-            dst.f_valid = src.f_valid
-        i += 1
-    d.entries = newitems
-    return False
-
-def ll_dict_remove_deleted_items(d):
-    new_allocated = _overallocate_entries_len(d.num_items)
-    if new_allocated < len(d.entries) // 2:
-        newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
-    else:
-        newitems = d.entries
-    #
-    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
-    isrc = 0
-    idst = 0
-    while isrc < len(d.entries):
-        if d.entries.valid(isrc):
-            src = d.entries[isrc]
-            dst = newitems[idst]
-            dst.key = src.key
-            dst.value = src.value
-            if hasattr(ENTRY, 'f_hash'):
-                dst.f_hash = src.f_hash
-            if hasattr(ENTRY, 'f_valid'):
-                assert src.f_valid
-                dst.f_valid = True
-            idst += 1
-        isrc += 1
-    d.entries = newitems
-    assert d.num_items == idst
-    d.num_used_items = idst
-
-    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
-
+def ll_dict_insertclean(d, key, value, hash):
+    # Internal routine used by ll_dict_resize() to insert an item which is
+    # known to be absent from the dict.  This routine also assumes that
+    # the dict contains no deleted entries.  This routine has the advantage
+    # of never calling d.keyhash() and d.keyeq(), so it cannot call back
+    # to user code.  ll_dict_insertclean() doesn't resize the dict, either.
+    i = ll_dict_lookup_clean(d, hash)
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    entry = d.entries[i]
+    entry.value = value
+    entry.key = key
+    if hasattr(ENTRY, 'f_hash'):     entry.f_hash = hash
+    if hasattr(ENTRY, 'f_valid'):    entry.f_valid = True
+    if hasattr(ENTRY, 'f_everused'): entry.f_everused = True
+    d.num_items += 1
+    d.resize_counter -= 3
 
 def ll_dict_delitem(d, key):
-    index = d.lookup_function(d, key, d.keyhash(key), FLAG_DELETE)
-    if index == -1:
+    i = ll_dict_lookup(d, key, d.keyhash(key))
+    if i & HIGHEST_BIT:
         raise KeyError
-    _ll_dict_del(d, index)
+    _ll_dict_del(d, i)
 
 @jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
-def _ll_dict_del(d, index):
-    d.entries.mark_deleted(index)
+def _ll_dict_del(d, i):
+    d.entries.mark_deleted(i)
     d.num_items -= 1
     # clear the key and the value if they are GC pointers
     ENTRIES = lltype.typeOf(d.entries).TO
     ENTRY = ENTRIES.OF
-    entry = d.entries[index]
+    entry = d.entries[i]
     if ENTRIES.must_clear_key:
         entry.key = lltype.nullptr(ENTRY.key.TO)
     if ENTRIES.must_clear_value:
@@ -645,179 +537,109 @@
     # avoid extra branches.
 
 def ll_dict_resize(d):
+    old_entries = d.entries
+    old_size = len(old_entries)
     # make a 'new_size' estimate and shrink it if there are many
     # deleted entry markers.  See CPython for why it is a good idea to
     # quadruple the dictionary size as long as it's not too big.
-    num_items = d.num_items
-    if num_items > 50000:
-        new_estimate = num_items * 2
-    else:
-        new_estimate = num_items * 4
+    num_items = d.num_items + 1
+    if num_items > 50000: new_estimate = num_items * 2
+    else:                 new_estimate = num_items * 4
     new_size = DICT_INITSIZE
     while new_size <= new_estimate:
         new_size *= 2
-
-    if new_size < _ll_len_of_d_indexes(d):
-        ll_dict_remove_deleted_items(d)
-    else:
-        ll_dict_reindex(d, new_size)
+    #
+    d.entries = lltype.typeOf(old_entries).TO.allocate(new_size)
+    d.num_items = 0
+    d.resize_counter = new_size * 2
+    i = 0
+    while i < old_size:
+        if old_entries.valid(i):
+            hash = old_entries.hash(i)
+            entry = old_entries[i]
+            ll_dict_insertclean(d, entry.key, entry.value, hash)
+        i += 1
+    old_entries.delete()
 ll_dict_resize.oopspec = 'dict.resize(d)'
 
-def ll_dict_reindex(d, new_size):
-    ll_malloc_indexes_and_choose_lookup(d, new_size)
-    d.resize_counter = new_size * 2 - d.num_items * 3
-    assert d.resize_counter > 0
-    #
-    insertcleanfn = ll_pick_insert_clean_function(d)
-    entries = d.entries
-    i = 0
-    while i < d.num_used_items:
-        if entries.valid(i):
-            hash = entries.hash(i)
-            insertcleanfn(d, hash, i)
-        i += 1
-    #old_entries.delete() XXXX!
-
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
-FREE = 0
-DELETED = 1
-VALID_OFFSET = 2
-MIN_INDEXES_MINUS_ENTRIES = VALID_OFFSET + 1
+ at jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key))
+def ll_dict_lookup(d, key, hash):
+    entries = d.entries
+    ENTRIES = lltype.typeOf(entries).TO
+    direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
+    mask = len(entries) - 1
+    i = r_uint(hash & mask)
+    # do the first try before any looping
+    if entries.valid(i):
+        checkingkey = entries[i].key
+        if direct_compare and checkingkey == key:
+            return i   # found the entry
+        if d.keyeq is not None and entries.hash(i) == hash:
+            # correct hash, maybe the key is e.g. a different pointer to
+            # an equal object
+            found = d.keyeq(checkingkey, key)
+            if d.paranoia:
+                if (entries != d.entries or
+                    not entries.valid(i) or entries[i].key != checkingkey):
+                    # the compare did major nasty stuff to the dict: start over
+                    return ll_dict_lookup(d, key, hash)
+            if found:
+                return i   # found the entry
+        freeslot = -1
+    elif entries.everused(i):
+        freeslot = intmask(i)
+    else:
+        return i | HIGHEST_BIT # pristine entry -- lookup failed
 
-FLAG_LOOKUP = 0
-FLAG_STORE = 1
-FLAG_DELETE = 2
-FLAG_DELETE_TRY_HARD = 3
-
-def new_lookup_functions(LOOKUP_FUNC, STORECLEAN_FUNC, T):
-    INDEXES = lltype.Ptr(lltype.GcArray(T))
-
-    def ll_kill_something(d):
-        i = 0
-        while True:
-            index = rffi.cast(lltype.Signed, d.indexes[i])
-            if index >= VALID_OFFSET:
-                d.indexes[i] = rffi.cast(T, DELETED)
-                return index
-            i += 1
-
-    @jit.look_inside_iff(lambda d, key, hash, store_flag:
-                         jit.isvirtual(d) and jit.isconstant(key))
-    def ll_dict_lookup(d, key, hash, store_flag):
-        entries = d.entries
-        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
-        mask = len(indexes) - 1
-        i = r_uint(hash & mask)
-        # do the first try before any looping
-        ENTRIES = lltype.typeOf(entries).TO
-        direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
-        index = rffi.cast(lltype.Signed, indexes[i])
-        if index >= VALID_OFFSET:
-            checkingkey = entries[index - VALID_OFFSET].key
+    # In the loop, a deleted entry (everused and not valid) is by far
+    # (factor of 100s) the least likely outcome, so test for that last.
+    perturb = r_uint(hash)
+    while 1:
+        # compute the next index using unsigned arithmetic
+        i = (i << 2) + i + perturb + 1
+        i = i & mask
+        # keep 'i' as a signed number here, to consistently pass signed
+        # arguments to the small helper methods.
+        if not entries.everused(i):
+            if freeslot == -1:
+                freeslot = intmask(i)
+            return r_uint(freeslot) | HIGHEST_BIT
+        elif entries.valid(i):
+            checkingkey = entries[i].key
             if direct_compare and checkingkey == key:
-                if store_flag == FLAG_DELETE:
-                    indexes[i] = rffi.cast(T, DELETED)
-                return index - VALID_OFFSET   # found the entry
-            if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
+                return i
+            if d.keyeq is not None and entries.hash(i) == hash:
                 # correct hash, maybe the key is e.g. a different pointer to
                 # an equal object
                 found = d.keyeq(checkingkey, key)
-                #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found)
                 if d.paranoia:
-                    XXX
-                    if (entries != d.entries or indexes != d.indexes or
-                        not entries.valid(ll_index_getitem(d.size, indexes, i))
-                        or entries.getitem_clean(index).key != checkingkey):
-                        # the compare did major nasty stuff to the dict: start over
-                        if d_signed_indexes(d):
-                            return ll_dict_lookup(d, key, hash,
-                                                  ll_index_getitem_signed)
-                        else:
-                            return ll_dict_lookup(d, key, hash,
-                                                  ll_index_getitem_int)
+                    if (entries != d.entries or
+                        not entries.valid(i) or entries[i].key != checkingkey):
+                        # the compare did major nasty stuff to the dict:
+                        # start over
+                        return ll_dict_lookup(d, key, hash)
                 if found:
-                    if store_flag == FLAG_DELETE:
-                        indexes[i] = rffi.cast(T, DELETED)
-                    return index - VALID_OFFSET
-            deletedslot = -1
-        elif index == DELETED:
-            deletedslot = i
-        else:
-            # pristine entry -- lookup failed
-            if store_flag == FLAG_STORE:
-                indexes[i] = rffi.cast(T, d.num_used_items + VALID_OFFSET)
-            elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
-                return ll_kill_something(d)
-            return -1
+                    return i   # found the entry
+        elif freeslot == -1:
+            freeslot = intmask(i)
+        perturb >>= PERTURB_SHIFT
 
-        # In the loop, a deleted entry (everused and not valid) is by far
-        # (factor of 100s) the least likely outcome, so test for that last.
-        perturb = r_uint(hash)
-        while 1:
-            # compute the next index using unsigned arithmetic
-            i = (i << 2) + i + perturb + 1
-            i = intmask(i) & mask
-            # keep 'i' as a signed number here, to consistently pass signed
-            # arguments to the small helper methods.
-            index = rffi.cast(lltype.Signed, indexes[i])
-            if index == FREE:
-                if store_flag == FLAG_STORE:
-                    if deletedslot == -1:
-                        deletedslot = i
-                    indexes[deletedslot] = rffi.cast(T, d.num_used_items +
-                                                     VALID_OFFSET)
-                elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
-                    return ll_kill_something(d)
-                return -1
-            elif index >= VALID_OFFSET:
-                checkingkey = entries[index - VALID_OFFSET].key
-                if direct_compare and checkingkey == key:
-                    if store_flag == FLAG_DELETE:
-                        indexes[i] = rffi.cast(T, DELETED)
-                    return index - VALID_OFFSET   # found the entry
-                if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
-                    # correct hash, maybe the key is e.g. a different pointer to
-                    # an equal object
-                    found = d.keyeq(checkingkey, key)
-                    if d.paranoia:
-                        XXX
-                        if (entries != d.entries or indexes != d.indexes or
-                            not entries.valid(ll_index_getitem(d.size, indexes, i)) or
-                            entries.getitem_clean(index).key != checkingkey):
-                            # the compare did major nasty stuff to the dict:
-                            # start over
-                            if d_signed_indexes(d):
-                                return ll_dict_lookup(d, key, hash,
-                                                      ll_index_getitem_signed)
-                            else:
-                                return ll_dict_lookup(d, key, hash,
-                                                      ll_index_getitem_int)
-                    if found:
-                        if store_flag == FLAG_DELETE:
-                            indexes[i] = rffi.cast(T, DELETED)
-                        return index - VALID_OFFSET
-            elif deletedslot == -1:
-                deletedslot = i
-            perturb >>= PERTURB_SHIFT
-
-    def ll_dict_store_clean(d, hash, index):
-        # a simplified version of ll_dict_lookup() which assumes that the
-        # key is new, and the dictionary doesn't contain deleted entries.
-        # It only finds the next free slot for the given hash.
-        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
-        mask = len(indexes) - 1
-        i = r_uint(hash & mask)
-        perturb = r_uint(hash)
-        while rffi.cast(lltype.Signed, indexes[i]) != 0:
-            i = (i << 2) + i + perturb + 1
-            i = i & mask
-            perturb >>= PERTURB_SHIFT
-        indexes[i] = rffi.cast(T, index + VALID_OFFSET)
-
-    return (llhelper(LOOKUP_FUNC, ll_dict_lookup),
-            llhelper(STORECLEAN_FUNC, ll_dict_store_clean))
+def ll_dict_lookup_clean(d, hash):
+    # a simplified version of ll_dict_lookup() which assumes that the
+    # key is new, and the dictionary doesn't contain deleted entries.
+    # It only finds the next free slot for the given hash.
+    entries = d.entries
+    mask = len(entries) - 1
+    i = r_uint(hash & mask)
+    perturb = r_uint(hash)
+    while entries.everused(i):
+        i = (i << 2) + i + perturb + 1
+        i = i & mask
+        perturb >>= PERTURB_SHIFT
+    return i
 
 # ____________________________________________________________
 #
@@ -827,23 +649,19 @@
 
 def ll_newdict(DICT):
     d = DICT.allocate()
-    d.entries = DICT.lookup_family.empty_array
-    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    d.entries = DICT.entries.TO.allocate(DICT_INITSIZE)
     d.num_items = 0
-    d.num_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
 
-def ll_newdict_size(DICT, orig_length_estimate):
-    length_estimate = (orig_length_estimate // 2) * 3
+def ll_newdict_size(DICT, length_estimate):
+    length_estimate = (length_estimate // 2) * 3
     n = DICT_INITSIZE
     while n < length_estimate:
         n *= 2
     d = DICT.allocate()
-    d.entries = DICT.entries.TO.allocate(orig_length_estimate)
-    ll_malloc_indexes_and_choose_lookup(d, n)
+    d.entries = DICT.entries.TO.allocate(n)
     d.num_items = 0
-    d.num_used_items = 0
     d.resize_counter = n * 2
     return d
 
@@ -882,17 +700,14 @@
 #
 #  Iteration.
 
-def get_ll_dictiter(DICTPTR):
-    return lltype.Ptr(lltype.GcStruct('dictiter',
-                                      ('dict', DICTPTR),
-                                      ('index', lltype.Signed)))
-
 class DictIteratorRepr(AbstractDictIteratorRepr):
 
     def __init__(self, r_dict, variant="keys"):
         self.r_dict = r_dict
         self.variant = variant
-        self.lowleveltype = get_ll_dictiter(r_dict.lowleveltype)
+        self.lowleveltype = lltype.Ptr(lltype.GcStruct('dictiter',
+                                         ('dict', r_dict.lowleveltype),
+                                         ('index', lltype.Signed)))
         self.ll_dictiter = ll_dictiter
         self.ll_dictnext = ll_dictnext_group[variant]
 
@@ -912,35 +727,31 @@
     def ll_dictnext(RETURNTYPE, iter):
         # note that RETURNTYPE is None for keys and values
         dict = iter.dict
-        if not dict:
-            raise StopIteration
-
-        entries = dict.entries
-        index = iter.index
-        assert index >= 0
-        entries_len = dict.num_used_items
-        while index < entries_len:
-            entry = entries[index]
-            is_valid = entries.valid(index)
-            index = index + 1
-            if is_valid:
-                iter.index = index
-                if RETURNTYPE is lltype.Void:
-                    return None
-                elif kind == 'items':
-                    r = lltype.malloc(RETURNTYPE.TO)
-                    r.item0 = recast(RETURNTYPE.TO.item0, entry.key)
-                    r.item1 = recast(RETURNTYPE.TO.item1, entry.value)
-                    return r
-                elif kind == 'keys':
-                    return entry.key
-                elif kind == 'values':
-                    return entry.value
-
-        # clear the reference to the dict and prevent restarts
-        iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
+        if dict:
+            entries = dict.entries
+            index = iter.index
+            assert index >= 0
+            entries_len = len(entries)
+            while index < entries_len:
+                entry = entries[index]
+                is_valid = entries.valid(index)
+                index = index + 1
+                if is_valid:
+                    iter.index = index
+                    if RETURNTYPE is lltype.Void:
+                        return None
+                    elif kind == 'items':
+                        r = lltype.malloc(RETURNTYPE.TO)
+                        r.item0 = recast(RETURNTYPE.TO.item0, entry.key)
+                        r.item1 = recast(RETURNTYPE.TO.item1, entry.value)
+                        return r
+                    elif kind == 'keys':
+                        return entry.key
+                    elif kind == 'values':
+                        return entry.value
+            # clear the reference to the dict and prevent restarts
+            iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
         raise StopIteration
-
     return ll_dictnext
 
 ll_dictnext_group = {'keys'  : _make_ll_dictnext('keys'),
@@ -950,77 +761,70 @@
 # _____________________________________________________________
 # methods
 
-def ll_dict_get(dict, key, default):
-    index = dict.lookup_function(dict, key, dict.keyhash(key), FLAG_LOOKUP)
-    if index == -1:
+def ll_get(dict, key, default):
+    i = ll_dict_lookup(dict, key, dict.keyhash(key))
+    if not i & HIGHEST_BIT:
+        return ll_get_value(dict, i)
+    else:
         return default
+
+def ll_setdefault(dict, key, default):
+    hash = dict.keyhash(key)
+    i = ll_dict_lookup(dict, key, hash)
+    if not i & HIGHEST_BIT:
+        return ll_get_value(dict, i)
     else:
-        return dict.entries[index].value
+        _ll_dict_setitem_lookup_done(dict, key, default, hash, i)
+        return default
 
-def ll_dict_setdefault(dict, key, default):
-    hash = dict.keyhash(key)
-    index = dict.lookup_function(dict, key, hash, FLAG_STORE)
-    if index == -1:
-        _ll_dict_setitem_lookup_done(dict, key, default, hash, -1)
-        return default
-    else:
-        return dict.entries[index].value
+def ll_copy(dict):
+    DICT = lltype.typeOf(dict).TO
+    dictsize = len(dict.entries)
+    d = DICT.allocate()
+    d.entries = DICT.entries.TO.allocate(dictsize)
+    d.num_items = dict.num_items
+    d.resize_counter = dict.resize_counter
+    if hasattr(DICT, 'fnkeyeq'):   d.fnkeyeq   = dict.fnkeyeq
+    if hasattr(DICT, 'fnkeyhash'): d.fnkeyhash = dict.fnkeyhash
+    i = 0
+    while i < dictsize:
+        d_entry = d.entries[i]
+        entry = dict.entries[i]
+        ENTRY = lltype.typeOf(d.entries).TO.OF
+        d_entry.key = entry.key
+        if hasattr(ENTRY, 'f_valid'):    d_entry.f_valid    = entry.f_valid
+        if hasattr(ENTRY, 'f_everused'): d_entry.f_everused = entry.f_everused
+        d_entry.value = entry.value
+        if hasattr(ENTRY, 'f_hash'):     d_entry.f_hash     = entry.f_hash
+        i += 1
+    return d
+ll_copy.oopspec = 'dict.copy(dict)'
 
-def ll_dict_copy(dict):
-    DICT = lltype.typeOf(dict).TO
-    newdict = DICT.allocate()
-    newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
+def ll_clear(d):
+    if (len(d.entries) == DICT_INITSIZE and
+        d.resize_counter == DICT_INITSIZE * 2):
+        return
+    old_entries = d.entries
+    d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_INITSIZE)
+    d.num_items = 0
+    d.resize_counter = DICT_INITSIZE * 2
+    old_entries.delete()
+ll_clear.oopspec = 'dict.clear(d)'
 
-    newdict.num_items = dict.num_items
-    newdict.num_used_items = dict.num_used_items
-    if hasattr(DICT, 'fnkeyeq'):
-        newdict.fnkeyeq = dict.fnkeyeq
-    if hasattr(DICT, 'fnkeyhash'):
-        newdict.fnkeyhash = dict.fnkeyhash
-
+def ll_update(dic1, dic2):
+    entries = dic2.entries
+    d2len = len(entries)
     i = 0
-    while i < newdict.num_used_items:
-        d_entry = newdict.entries[i]
-        entry = dict.entries[i]
-        ENTRY = lltype.typeOf(newdict.entries).TO.OF
-        d_entry.key = entry.key
-        if hasattr(ENTRY, 'f_valid'):
-            d_entry.f_valid = entry.f_valid
-        d_entry.value = entry.value
-        if hasattr(ENTRY, 'f_hash'):
-            d_entry.f_hash = entry.f_hash
-        i += 1
-
-    ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
-    return newdict
-ll_dict_copy.oopspec = 'dict.copy(dict)'
-
-def ll_dict_clear(d):
-    if d.num_used_items == 0:
-        return
-    DICT = lltype.typeOf(d).TO
-    old_entries = d.entries
-    d.entries = DICT.lookup_family.empty_array
-    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
-    d.num_items = 0
-    d.num_used_items = 0
-    d.resize_counter = DICT_INITSIZE * 2
-    # old_entries.delete() XXX
-ll_dict_clear.oopspec = 'dict.clear(d)'
-
-def ll_dict_update(dic1, dic2):
-    i = 0
-    while i < dic2.num_used_items:
-        entries = dic2.entries
+    while i < d2len:
         if entries.valid(i):
             entry = entries[i]
             hash = entries.hash(i)
             key = entry.key
             value = entry.value
-            index = dic1.lookup_function(dic1, key, hash, FLAG_STORE)
-            _ll_dict_setitem_lookup_done(dic1, key, value, hash, index)
+            j = ll_dict_lookup(dic1, key, hash)
+            _ll_dict_setitem_lookup_done(dic1, key, value, hash, j)
         i += 1
-ll_dict_update.oopspec = 'dict.update(dic1, dic2)'
+ll_update.oopspec = 'dict.update(dic1, dic2)'
 
 # this is an implementation of keys(), values() and items()
 # in a single function.
@@ -1037,7 +841,7 @@
     def ll_kvi(LIST, dic):
         res = LIST.ll_newlist(dic.num_items)
         entries = dic.entries
-        dlen = dic.num_used_items
+        dlen = len(entries)
         items = res.ll_items()
         i = 0
         p = 0
@@ -1066,34 +870,38 @@
 ll_dict_values = _make_ll_keys_values_items('values')
 ll_dict_items  = _make_ll_keys_values_items('items')
 
-def ll_dict_contains(d, key):
-    i = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
-    return i != -1
+def ll_contains(d, key):
+    i = ll_dict_lookup(d, key, d.keyhash(key))
+    return not i & HIGHEST_BIT
+
+POPITEMINDEX = lltype.Struct('PopItemIndex', ('nextindex', lltype.Signed))
+global_popitem_index = lltype.malloc(POPITEMINDEX, zero=True, immortal=True)
 
 def _ll_getnextitem(dic):
-    if dic.num_items == 0:
-        raise KeyError
-
     entries = dic.entries
-
-    i = dic.num_used_items - 1
-    while True:
+    ENTRY = lltype.typeOf(entries).TO.OF
+    dmask = len(entries) - 1
+    if hasattr(ENTRY, 'f_hash'):
+        if entries.valid(0):
+            return 0
+        base = entries[0].f_hash
+    else:
+        base = global_popitem_index.nextindex
+    counter = 0
+    while counter <= dmask:
+        i = (base + counter) & dmask
+        counter += 1
         if entries.valid(i):
             break
-        i -= 1
+    else:
+        raise KeyError
+    if hasattr(ENTRY, 'f_hash'):
+        entries[0].f_hash = base + counter
+    else:
+        global_popitem_index.nextindex = base + counter
+    return i
 
-    key = entries[i].key
-    index = dic.lookup_function(dic, key, entries.hash(i),
-                                FLAG_DELETE_TRY_HARD)
-    # if the lookup function returned me a random strange thing,
-    # don't care about deleting the item
-    if index == dic.num_used_items - 1:
-        dic.num_used_items -= 1
-    else:
-        assert index != -1
-    return index
-
-def ll_dict_popitem(ELEM, dic):
+def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
     r = lltype.malloc(ELEM.TO)
@@ -1102,18 +910,17 @@
     _ll_dict_del(dic, r_uint(i))
     return r
 
-def ll_dict_pop(dic, key):
-    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
-    if index == -1:
+def ll_pop(dic, key):
+    i = ll_dict_lookup(dic, key, dic.keyhash(key))
+    if not i & HIGHEST_BIT:
+        value = ll_get_value(dic, r_uint(i))
+        _ll_dict_del(dic, r_uint(i))
+        return value
+    else:
         raise KeyError
-    value = dic.entries[index].value
-    _ll_dict_del(dic, index)
-    return value
 
-def ll_dict_pop_default(dic, key, dfl):
-    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
-    if index == -1:
+def ll_pop_default(dic, key, dfl):
+    try:
+        return ll_pop(dic, key)
+    except KeyError:
         return dfl
-    value = dic.entries[index].value
-    _ll_dict_del(dic, index)
-    return value

From noreply at buildbot.pypy.org  Thu Oct 10 17:13:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 10 Oct 2013 17:13:26 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: hg merge default
Message-ID: <20131010151326.638C51C02D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67299:b908aee5cb95
Date: 2013-10-10 17:12 +0200
http://bitbucket.org/pypy/pypy/changeset/b908aee5cb95/

Log:	hg merge default

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -10,8 +10,35 @@
 
 import os
 def get_include():
-    head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.join(head, '../include')
+    """
+    Return the directory that contains the NumPy \\*.h header files.
+
+    Extension modules that need to compile against NumPy should use this
+    function to locate the appropriate include directory.
+
+    Notes
+    -----
+    When using ``distutils``, for example in ``setup.py``.
+    ::
+
+        import numpy as np
+        ...
+        Extension('extension_name', ...
+                include_dirs=[np.get_include()])
+        ...
+
+    """
+    import numpy
+    if getattr(numpy, 'show_config', None) is None:
+        # running from numpy source directory
+        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
+        return os.path.join(head, '../include')
+    else:
+        # using installed numpy core headers
+        import numpy.core as core
+        d = os.path.join(os.path.dirname(core.__file__), 'include')
+    return d
+
 
 
 __all__ = ['__version__', 'get_include']
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -52,6 +52,9 @@
 .. branch: ndarray-subtype
 Allow subclassing ndarray, i.e. matrix
 
+.. branch: ndarray-sort
+Implement ndarray in-place sorting (for numeric types, no non-native byte order)
+
 .. branch: pypy-pyarray
 Implement much of numpy's c api in cpyext, allows (slow) access to ndarray
 from c
@@ -87,6 +90,7 @@
 .. branch: no-release-gil
 .. branch: safe-win-mmap
 .. branch: boolean-indexing-cleanup
+.. branch: cpyyest-best_base
 
 .. branch: nobold-backtrace
 Work on improving UnionError messages and stack trace displays.
@@ -103,3 +107,5 @@
 
 .. branch: file-support-in-rpython
 make open() and friends rpython
+
+
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -99,7 +99,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii'])
+                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
@@ -196,7 +196,7 @@
         assert PyUnicode_GetDefaultEncoding() == 'ascii'
 
 class AppTestCpythonExtensionBase(LeakCheckingTest):
-    
+
     def setup_class(cls):
         cls.space.getbuiltinmodule("cpyext")
         from pypy.module.imp.importing import importhook
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -156,7 +156,7 @@
             def __init__(self):
                 self.foobar = 32
                 super(UnicodeSubclass2, self).__init__()
-        
+
         newobj = UnicodeSubclass2()
         assert newobj.get_val() == 42
         assert newobj.foobar == 32
@@ -358,6 +358,13 @@
         assert w_obj is None
         assert api.PyErr_Occurred() is None
 
+    def test_ndarray_ref(self, space, api):
+        w_obj = space.appexec([], """():
+            import numpypy as np
+            return np.int64(2)""")
+        ref = make_ref(space, w_obj)
+        api.Py_DecRef(ref)
+
 class AppTestSlots(AppTestCpythonExtensionBase):
     def test_some_slots(self):
         module = self.import_extension('foo', [
@@ -525,7 +532,7 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
-        
+
     def test_bool(self):
         module = self.import_extension('foo', [
             ("newInt", "METH_VARARGS",
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -4,7 +4,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.annlowlevel import llhelper
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
-from pypy.objspace.std.typeobject import W_TypeObject
+from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
@@ -574,33 +574,7 @@
 def best_base(space, bases_w):
     if not bases_w:
         return None
-
-    w_winner = None
-    w_base = None
-    for w_base_i in bases_w:
-        if isinstance(w_base_i, W_ClassObject):
-            # old-style base
-            continue
-        assert isinstance(w_base_i, W_TypeObject)
-        w_candidate = solid_base(space, w_base_i)
-        if not w_winner:
-            w_winner = w_candidate
-            w_base = w_base_i
-        elif space.abstract_issubclass_w(w_winner, w_candidate):
-            pass
-        elif space.abstract_issubclass_w(w_candidate, w_winner):
-            w_winner = w_candidate
-            w_base = w_base_i
-        else:
-            raise OperationError(
-                space.w_TypeError,
-                space.wrap("multiple bases have instance lay-out conflict"))
-    if w_base is None:
-        raise OperationError(
-            space.w_TypeError,
-                space.wrap("a new-style class can't have only classic bases"))
-
-    return w_base
+    return find_best_base(space, bases_w)
 
 def inherit_slots(space, pto, w_base):
     # XXX missing: nearly everything
diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -356,6 +356,10 @@
         from pypy.module.micronumpy.arrayimpl.sort import argsort_array
         return argsort_array(self, space, w_axis)
 
+    def sort(self, space, w_axis, w_order):
+        from pypy.module.micronumpy.arrayimpl.sort import sort_array
+        return sort_array(self, space, w_axis, w_order)
+
     def base(self):
         return None
 
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -17,7 +17,7 @@
 
 INT_SIZE = rffi.sizeof(lltype.Signed)
 
-def make_sort_function(space, itemtype, comp_type, count=1):
+def make_argsort_function(space, itemtype, comp_type, count=1):
     TP = itemtype.T
     step = rffi.sizeof(TP)
 
@@ -137,8 +137,8 @@
         else:
             shape = arr.get_shape()
             if axis < 0:
-                axis = len(shape) + axis - 1
-            if axis < 0 or axis > len(shape):
+                axis = len(shape) + axis
+            if axis < 0 or axis >= len(shape):
                 raise OperationError(space.w_IndexError, space.wrap(
                                                     "Wrong axis %d" % axis))
             iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
@@ -162,7 +162,7 @@
     return argsort
 
 def argsort_array(arr, space, w_axis):
-    cache = space.fromcache(SortCache) # that populates SortClasses
+    cache = space.fromcache(ArgSortCache) # that populates ArgSortClasses
     itemtype = arr.dtype.itemtype
     for tp in all_types:
         if isinstance(itemtype, tp[0]):
@@ -178,6 +178,166 @@
 all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
 all_types = unrolling_iterable(all_types)
 
+def make_sort_function(space, itemtype, comp_type, count=1):
+    TP = itemtype.T
+    step = rffi.sizeof(TP)
+
+    class Repr(object):
+        def __init__(self, stride_size, size, values, start):
+            self.stride_size = stride_size
+            self.start = start
+            self.size = size
+            self.values = values
+
+        def getitem(self, item):
+            if count < 2:
+                v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                                    + self.start)
+            else:
+                v = []
+                for i in range(count):
+                    _v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                                    + self.start + step * i)
+                    v.append(_v)
+            if comp_type == 'int':
+                v = intmask(v)
+            elif comp_type == 'float':
+                v = float(v)
+            elif comp_type == 'complex':
+                v = [float(v[0]),float(v[1])]
+            else:
+                raise NotImplementedError('cannot reach')
+            return (v)
+
+        def setitem(self, idx, item):
+            if count < 2:
+                raw_storage_setitem(self.values, idx * self.stride_size +
+                                self.start, rffi.cast(TP, item))
+            else:
+                i = 0
+                for val in item:
+                    raw_storage_setitem(self.values, idx * self.stride_size +
+                                self.start + i*step, rffi.cast(TP, val))
+                    i += 1
+
+    class ArgArrayRepWithStorage(Repr):
+        def __init__(self, stride_size, size):
+            start = 0
+            values = alloc_raw_storage(size * stride_size,
+                                            track_allocation=False)
+            Repr.__init__(self, stride_size,
+                          size, values, start)
+
+        def __del__(self):
+            free_raw_storage(self.values, track_allocation=False)
+
+    def arg_getitem(lst, item):
+        return lst.getitem(item)
+
+    def arg_setitem(lst, item, value):
+        lst.setitem(item, value)
+
+    def arg_length(lst):
+        return lst.size
+
+    def arg_getitem_slice(lst, start, stop):
+        retval = ArgArrayRepWithStorage(lst.stride_size, stop-start)
+        for i in range(stop-start):
+            retval.setitem(i, lst.getitem(i+start))
+        return retval
+
+    if count < 2:
+        def arg_lt(a, b):
+            # handles NAN and INF
+            return a < b or b != b and a == a
+    else:
+        def arg_lt(a, b):
+            for i in range(count):
+                if b[i] != b[i] and a[i] == a[i]:
+                    return True
+                elif b[i] == b[i] and a[i] != a[i]:
+                    return False
+            for i in range(count):
+                if a[i] < b[i]:
+                    return True
+                elif a[i] > b[i]:
+                    return False
+            # Does numpy do True?
+            return False
+
+    ArgSort = make_timsort_class(arg_getitem, arg_setitem, arg_length,
+                                 arg_getitem_slice, arg_lt)
+
+    def sort(arr, space, w_axis, itemsize):
+        if w_axis is space.w_None:
+            # note that it's fine to pass None here as we're not going
+            # to pass the result around (None is the link to base in slices)
+            arr = arr.reshape(space, None, [arr.get_size()])
+            axis = 0
+        elif w_axis is None:
+            axis = -1
+        else:
+            axis = space.int_w(w_axis)
+        # create array of indexes
+        if len(arr.get_shape()) == 1:
+            r = Repr(itemsize, arr.get_size(), arr.get_storage(),
+                     arr.start)
+            ArgSort(r).sort()
+        else:
+            shape = arr.get_shape()
+            if axis < 0:
+                axis = len(shape) + axis
+            if axis < 0 or axis >= len(shape):
+                raise OperationError(space.w_IndexError, space.wrap(
+                                                    "Wrong axis %d" % axis))
+            iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
+            iter = AxisIterator(arr, iterable_shape, axis, False)
+            stride_size = arr.strides[axis]
+            axis_size = arr.shape[axis]
+            while not iter.done():
+                r = Repr(stride_size, axis_size, arr.get_storage(), iter.offset)
+                ArgSort(r).sort()
+                iter.next()
+
+    return sort
+
+def sort_array(arr, space, w_axis, w_order):
+    cache = space.fromcache(SortCache) # that populates SortClasses
+    itemtype = arr.dtype.itemtype
+    if not arr.dtype.native:
+        raise OperationError(space.w_NotImplementedError,
+           space.wrap("sorting of non-native btyeorder not supported yet"))
+    for tp in all_types:
+        if isinstance(itemtype, tp[0]):
+            return cache._lookup(tp)(arr, space, w_axis,
+                                     itemtype.get_element_size())
+    # XXX this should probably be changed
+    raise OperationError(space.w_NotImplementedError,
+           space.wrap("sorting of non-numeric types " + \
+                  "'%s' is not implemented" % arr.dtype.get_name(), ))
+
+all_types = (types.all_float_types + types.all_complex_types +
+             types.all_int_types)
+all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
+all_types = unrolling_iterable(all_types)
+
+class ArgSortCache(object):
+    built = False
+
+    def __init__(self, space):
+        if self.built:
+            return
+        self.built = True
+        cache = {}
+        for cls, it in all_types._items:
+            if it == 'complex':
+                cache[cls] = make_argsort_function(space, cls, it, 2)
+            else:
+                cache[cls] = make_argsort_function(space, cls, it)
+        self.cache = cache
+        self._lookup = specialize.memo()(lambda tp : cache[tp[0]])
+
+
 class SortCache(object):
     built = False
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -629,9 +629,13 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "setflags not implemented yet"))
 
-    def descr_sort(self, space, w_axis=-1, w_kind='quicksort', w_order=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "sort not implemented yet"))
+    @unwrap_spec(kind=str)
+    def descr_sort(self, space, w_axis=None, kind='quicksort', w_order=None):
+        # happily ignore the kind
+        # modify the array in-place
+        if self.is_scalar():
+            return
+        return self.implementation.sort(space, w_axis, w_order)
 
     def descr_squeeze(self, space):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -1118,6 +1122,7 @@
     conj = interp2app(W_NDimArray.descr_conj),
 
     argsort  = interp2app(W_NDimArray.descr_argsort),
+    sort  = interp2app(W_NDimArray.descr_sort),
     astype   = interp2app(W_NDimArray.descr_astype),
     base     = GetSetProperty(W_NDimArray.descr_get_base),
     byteswap = interp2app(W_NDimArray.descr_byteswap),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2652,55 +2652,6 @@
         assert array([1, 2, 3], '>i2')[::2].tostring() == '\x00\x01\x00\x03'
         assert array(0, dtype='i2').tostring() == '\x00\x00'
 
-    def test_argsort_dtypes(self):
-        from numpypy import array, arange
-        assert array(2.0).argsort() == 0
-        nnp = self.non_native_prefix
-        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
-                        nnp + 'i2', complex]:
-            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
-            c = a.copy()
-            res = a.argsort()
-            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
-                'a,res,dtype %r,%r,%r' % (a,res,dtype)
-            assert (a == c).all() # not modified
-            a = arange(100)
-            assert (a.argsort() == a).all()
-        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
-
-    def test_argsort_nd(self):
-        from numpypy import array
-        a = array([[4, 2], [1, 3]])
-        assert (a.argsort() == [[1, 0], [0, 1]]).all()
-        a = array(range(10) + range(10) + range(10))
-        b = a.argsort()
-        assert (b[:3] == [0, 10, 20]).all()
-        #trigger timsort 'run' mode which calls arg_getitem_slice
-        a = array(range(100) + range(100) + range(100))
-        b = a.argsort()
-        assert (b[:3] == [0, 100, 200]).all()
-        a = array([[[]]]).reshape(3,4,0)
-        b = a.argsort()
-        assert b.size == 0
-
-    def test_argsort_random(self):
-        from numpypy import array
-        from _random import Random
-        rnd = Random(1)
-        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
-        a.argsort()
-
-    def test_argsort_axis(self):
-        from numpypy import array
-        a = array([[4, 2], [1, 3]])
-        assert (a.argsort(axis=None) == [2, 1, 3, 0]).all()
-        assert (a.argsort(axis=-1) == [[1, 0], [0, 1]]).all()
-        assert (a.argsort(axis=0) == [[1, 0], [0, 1]]).all()
-        assert (a.argsort(axis=1) == [[1, 0], [0, 1]]).all()
-        a = array([[3, 2, 1], [1, 2, 3]])
-        assert (a.argsort(axis=0) == [[1, 0, 0], [0, 1, 1]]).all()
-        assert (a.argsort(axis=1) == [[2, 1, 0], [0, 1, 2]]).all()
-
 
 class AppTestRanges(BaseNumpyAppTest):
     def test_arange(self):
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -0,0 +1,322 @@
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestSupport(BaseNumpyAppTest):
+    def setup_class(cls):
+        import struct
+        BaseNumpyAppTest.setup_class.im_func(cls)
+        cls.w_data = cls.space.wrap(struct.pack('dddd', 1, 2, 3, 4))
+        cls.w_fdata = cls.space.wrap(struct.pack('f', 2.3))
+        cls.w_float16val = cls.space.wrap('\x00E') # 5.0 in float16
+        cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
+        cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
+        cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+
+    def test_argsort_dtypes(self):
+        from numpypy import array, arange
+        assert array(2.0).argsort() == 0
+        nnp = self.non_native_prefix
+        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
+                        nnp + 'i2', complex]:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            c = a.copy()
+            res = a.argsort()
+            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
+                'a,res,dtype %r,%r,%r' % (a,res,dtype)
+            assert (a == c).all() # not modified
+            a = arange(100)
+            assert (a.argsort() == a).all()
+        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
+
+    def test_argsort_nd(self):
+        from numpypy import array
+        a = array([[4, 2], [1, 3]])
+        assert (a.argsort() == [[1, 0], [0, 1]]).all()
+        a = array(range(10) + range(10) + range(10))
+        b = a.argsort()
+        assert (b[:3] == [0, 10, 20]).all()
+        #trigger timsort 'run' mode which calls arg_getitem_slice
+        a = array(range(100) + range(100) + range(100))
+        b = a.argsort()
+        assert (b[:3] == [0, 100, 200]).all()
+        a = array([[[]]]).reshape(3,4,0)
+        b = a.argsort()
+        assert b.size == 0
+
+    def test_argsort_random(self):
+        from numpypy import array
+        from _random import Random
+        rnd = Random(1)
+        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
+        a.argsort()
+
+    def test_argsort_axis(self):
+        from numpypy import array
+        a = array([[4, 2], [1, 3]])
+        assert (a.argsort(axis=None) == [2, 1, 3, 0]).all()
+        assert (a.argsort(axis=-1) == [[1, 0], [0, 1]]).all()
+        assert (a.argsort(axis=0) == [[1, 0], [0, 1]]).all()
+        assert (a.argsort(axis=1) == [[1, 0], [0, 1]]).all()
+        a = array([[3, 2, 1], [1, 2, 3]])
+        assert (a.argsort(axis=0) == [[1, 0, 0], [0, 1, 1]]).all()
+        assert (a.argsort(axis=1) == [[2, 1, 0], [0, 1, 2]]).all()
+
+    def test_sort_dtypes(self):
+        from numpypy import array, arange
+        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
+                        'i2', complex]:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
+            c = a.copy()
+            a.sort()
+            assert (a == b).all(), \
+                'a,orig,dtype %r,%r,%r' % (a,c,dtype)
+        a = arange(100)
+        c = a.copy()
+        a.sort()
+        assert (a == c).all()
+
+    def test_sort_dtypesi_nonnative(self):
+        from numpypy import array
+        nnp = self.non_native_prefix
+        for dtype in [ nnp + 'i2']:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
+            c = a.copy()
+            exc = raises(NotImplementedError, a.sort)
+            assert exc.value[0].find('supported') >= 0
+            #assert (a == b).all(), \
+            #    'a,orig,dtype %r,%r,%r' % (a,c,dtype)
+
+
+# tests from numpy/tests/test_multiarray.py
+    def test_sort_corner_cases(self):
+        # test ordering for floats and complex containing nans. It is only
+        # necessary to check the lessthan comparison, so sorts that
+        # only follow the insertion sort path are sufficient. We only
+        # test doubles and complex doubles as the logic is the same.
+
+        # check doubles
+        from numpypy import array, nan, zeros, complex128, arange
+        from numpy import isnan
+        a = array([nan, 1, 0])
+        b = a.copy()
+        b.sort()
+        assert (isnan(b) == isnan(a[::-1])).all()
+        assert (b[:2] == a[::-1][:2]).all()
+
+        # check complex
+        a = zeros(9, dtype=complex128)
+        a.real += [nan, nan, nan, 1, 0, 1, 1, 0, 0]
+        a.imag += [nan, 1, 0, nan, nan, 1, 0, 1, 0]
+        b = a.copy()
+        b.sort()
+        assert (isnan(b) == isnan(a[::-1])).all()
+        assert (b[:4] == a[::-1][:4]).all()
+
+        # all c scalar sorts use the same code with different types
+        # so it suffices to run a quick check with one type. The number
+        # of sorted items must be greater than ~50 to check the actual
+        # algorithm because quick and merge sort fall over to insertion
+        # sort for small arrays.
+        a = arange(101)
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h'] :
+            msg = "scalar sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+        # test complex sorts. These use the same code as the scalars
+        # but the compare fuction differs.
+        ai = a*1j + 1
+        bi = b*1j + 1
+        for kind in ['q', 'm', 'h'] :
+            msg = "complex sort, real part == 1, kind=%s" % kind
+            c = ai.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+            c = bi.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+        ai = a + 1j
+        bi = b + 1j
+        for kind in ['q', 'm', 'h'] :
+            msg = "complex sort, imag part == 1, kind=%s" % kind
+            c = ai.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+            c = bi.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+
+        # check axis handling. This should be the same for all type
+        # specific sorts, so we only check it for one type and one kind
+        a = array([[3, 2], [1, 0]])
+        b = array([[1, 0], [3, 2]])
+        c = array([[2, 3], [0, 1]])
+        d = a.copy()
+        d.sort(axis=0)
+        assert (d == b).all(), "test sort with axis=0"
+        d = a.copy()
+        d.sort(axis=1)
+        assert (d == c).all(), "test sort with axis=1"
+        d = a.copy()
+        d.sort()
+        assert (d == c).all(), "test sort with default axis"
+
+    def test_sort_corner_cases_string_records(self):
+        skip('not implemented yet')
+        from numpypy import array, dtype
+        # test string sorts.
+        s = 'aaaaaaaa'
+        a = array([s + chr(i) for i in range(101)])
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h'] :
+            msg = "string sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+
+        # test record array sorts.
+        dt =dtype([('f', float), ('i', int)])
+        a = array([(i, i) for i in range(101)], dtype = dt)
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "object sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_unicode(self):
+        from numpypy import array
+        # test unicode sorts.
+        s = 'aaaaaaaa'
+        try:
+            a = array([s + chr(i) for i in range(101)], dtype=unicode)
+            b = a[::-1].copy()
+        except:
+            skip('unicode type not supported yet')
+        for kind in ['q', 'm', 'h'] :
+            msg = "unicode sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_objects(self):
+        # test object array sorts.
+        from numpypy import empty
+        try:
+            a = empty((101,), dtype=object)
+        except:
+            skip('object type not supported yet')
+        a[:] = list(range(101))
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "object sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_datetime(self):
+        from numpypy import arange
+        # test datetime64 sorts.
+        try:
+            a = arange(0, 101, dtype='datetime64[D]')
+        except:
+            skip('datetime type not supported yet')
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "datetime64 sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+        # test timedelta64 sorts.
+        a = arange(0, 101, dtype='timedelta64[D]')
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "timedelta64 sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_order(self):
+        from numpypy import array, zeros
+        from sys import byteorder
+        # Test sorting an array with fields
+        skip('not implemented yet')
+        x1 = array([21, 32, 14])
+        x2 = array(['my', 'first', 'name'])
+        x3=array([3.1, 4.5, 6.2])
+        r=zeros(3, dtype=[('id','i'),('word','S5'),('number','f')])
+        r['id'] = x1
+        r['word'] = x2
+        r['number'] = x3
+
+        r.sort(order=['id'])
+        assert (r['id'] == [14, 21, 32]).all()
+        assert (r['word'] == ['name', 'my', 'first']).all()
+        assert max(abs(r['number'] - [6.2, 3.1, 4.5])) < 1e-6
+
+        r.sort(order=['word'])
+        assert (r['id'] == [32, 21, 14]).all()
+        assert (r['word'] == ['first', 'my', 'name']).all()
+        assert max(abs(r['number'] - [4.5, 3.1, 6.2])) < 1e-6
+
+        r.sort(order=['number'])
+        assert (r['id'] == [21, 32, 14]).all()
+        assert (r['word'] == ['my', 'first', 'name']).all()
+        assert max(abs(r['number'] - [3.1, 4.5, 6.2])) < 1e-6
+
+        if byteorder == 'little':
+            strtype = '>i2'
+        else:
+            strtype = '<i2'
+        mydtype = [('name', 'S5'), ('col2', strtype)]
+        r = array([('a', 1), ('b', 255), ('c', 3), ('d', 258)],
+                     dtype= mydtype)
+        r.sort(order='col2')
+        assert (r['col2'] == [1, 3, 255, 258]).all()
+        assert (r == array([('a', 1), ('c', 3), ('b', 255), ('d', 258)],
+                                 dtype=mydtype)).all()
+
+
+# tests from numpy/tests/test_regression.py
+    def test_sort_bigendian(self):
+        skip('not implemented yet')
+        from numpypy import array, dtype
+        a = array(range(11),dtype='float64')
+        c = a.astype(dtype('<f8'))
+        c.sort()
+        assert max(abs(a-c)) < 1e-32
+
+    def test_string_sort_with_zeros(self):
+        skip('not implemented yet')
+        from numpypy import fromstring
+        """Check sort for strings containing zeros."""
+        x = fromstring("\x00\x02\x00\x01", dtype="S2")
+        y = fromstring("\x00\x01\x00\x02", dtype="S2")
+        x.sort(kind='q')
+        assert (x == y).all()
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1864,7 +1864,6 @@
         return interp_boxes.W_StringBox(arr,  0, arr.dtype)
 
     def fill(self, storage, width, box, start, stop, offset):
-        from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArrayNotOwning
         for i in xrange(start, stop, width):
             self._store(storage, i, offset, box)
 
@@ -1873,6 +1872,13 @@
 class UnicodeType(BaseType, BaseStringType):
     T = lltype.UniChar
 
+    @jit.unroll_safe
+    def coerce(self, space, dtype, w_item):
+        if isinstance(w_item, interp_boxes.W_UnicodeBox):
+            return w_item
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "coerce (probably from set_item) not implemented for unicode type"))
+
 NonNativeUnicodeType = UnicodeType
 
 class VoidType(BaseType, BaseStringType):
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -50,7 +50,6 @@
         'lstat': 'interp_posix.lstat',
         'stat_float_times': 'interp_posix.stat_float_times',
 
-        'ctermid': 'interp_posix.ctermid',
         'dup': 'interp_posix.dup',
         'dup2': 'interp_posix.dup2',
         'access': 'interp_posix.access',
@@ -150,6 +149,8 @@
         interpleveldefs['nice'] = 'interp_posix.nice'
     if hasattr(os, 'getlogin'):
         interpleveldefs['getlogin'] = 'interp_posix.getlogin'
+    if hasattr(os, 'ctermid'):
+        interpleveldefs['ctermid'] = 'interp_posix.ctermid'
 
     for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid',
                  'seteuid', 'setgid', 'setegid', 'getgroups', 'getpgrp',
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -1,5 +1,5 @@
 import operator
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.objspace.std import model, newformat
 from pypy.objspace.std.floattype import float_typedef, W_AbstractFloatObject
 from pypy.objspace.std.multimethod import FailedToImplementArgs
@@ -424,7 +424,16 @@
     x = w_float1.floatval
     y = w_float2.floatval
 
-    return W_FloatObject(_pow(space, x, y))
+    try:
+        result = _pow(space, x, y)
+    except PowDomainError:
+        raise operationerrfmt(space.w_ValueError,
+                              "negative number cannot be raised to a "
+                              "fractional power")
+    return W_FloatObject(result)
+
+class PowDomainError(ValueError):
+    """Signals a negative number raised to a fractional power"""
 
 def _pow(space, x, y):
     # Sort out special cases here instead of relying on pow()
@@ -478,16 +487,14 @@
                                             "a negative power"))
 
     negate_result = False
-    # special case: "(-1.0) ** bignum" should not raise ValueError,
+    # special case: "(-1.0) ** bignum" should not raise PowDomainError,
     # unlike "math.pow(-1.0, bignum)".  See http://mail.python.org/
     # -           pipermail/python-bugs-list/2003-March/016795.html
     if x < 0.0:
         if isnan(y):
             return NAN
         if math.floor(y) != y:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("negative number cannot be "
-                                            "raised to a fractional power"))
+            raise PowDomainError
         # y is an exact integer, albeit perhaps a very large one.
         # Replace x by its absolute value and remember to negate the
         # pow result if y is odd.
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -30,7 +30,7 @@
 from rpython.rlib import debug, jit, rerased
 from rpython.rlib.listsort import make_timsort_class
 from rpython.rlib.objectmodel import (
-    instantiate, newlist_hint, resizelist_hint, specialize)
+    instantiate, newlist_hint, resizelist_hint, specialize, import_from_mixin)
 from rpython.tool.sourcetools import func_with_new_name
 
 __all__ = ['W_ListObject', 'make_range_list', 'make_empty_list_with_size']
@@ -1170,7 +1170,6 @@
 
 
 class AbstractUnwrappedStrategy(object):
-    _mixin_ = True
 
     def wrap(self, unwrapped):
         raise NotImplementedError
@@ -1329,7 +1328,6 @@
 
     def setslice(self, w_list, start, step, slicelength, w_other):
         assert slicelength >= 0
-        items = self.unerase(w_list.lstorage)
 
         if self is self.space.fromcache(ObjectListStrategy):
             w_other = w_other._temporarily_as_objects()
@@ -1341,6 +1339,7 @@
             w_list.setslice(start, step, slicelength, w_other_as_object)
             return
 
+        items = self.unerase(w_list.lstorage)
         oldsize = len(items)
         len2 = w_other.length()
         if step == 1:  # Support list resizing for non-extended slices
@@ -1456,7 +1455,9 @@
         self.unerase(w_list.lstorage).reverse()
 
 
-class ObjectListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class ObjectListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "object"
 
@@ -1489,7 +1490,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class IntegerListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class IntegerListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0
     _applevel_repr = "int"
 
@@ -1520,7 +1523,30 @@
         return self.unerase(w_list.lstorage)
 
 
-class FloatListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            l = self.unerase(w_list.lstorage)
+            other = w_other.getitems_int()
+            assert other is not None
+            l += other
+            return
+        return self._base_extend_from_list(w_list, w_other)
+
+
+    _base_setslice = setslice
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            storage = self.erase(w_other.getitems_int())
+            w_other = W_ListObject.from_storage_and_strategy(
+                    self.space, storage, self)
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
+class FloatListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0.0
     _applevel_repr = "float"
 
@@ -1548,7 +1574,9 @@
             l.reverse()
 
 
-class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class StringListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "str"
 
@@ -1579,7 +1607,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class UnicodeListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class UnicodeListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "unicode"
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -225,6 +225,15 @@
         l.setslice(0, 1, 2, W_ListObject(space, [w('a'), w(2), w(3)]))
         assert isinstance(l.strategy, ObjectListStrategy)
 
+    def test_setslice_int_range(self):
+        space = self.space
+        w = space.wrap
+        l = W_ListObject(space, [w(1), w(2), w(3)])
+        assert isinstance(l.strategy, IntegerListStrategy)
+        l.setslice(0, 1, 2, make_range_list(space, 5, 1, 4))
+        assert isinstance(l.strategy, IntegerListStrategy)
+
+
     def test_setslice_List(self):
         space = self.space
 
@@ -467,6 +476,12 @@
         l4 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3), self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         assert self.space.eq_w(l3, l4)
 
+    def test_add_of_range_and_int(self):
+        l1 = make_range_list(self.space, 0, 1, 100)
+        l2 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
+        l3 = self.space.add(l2, l1)
+        assert l3.strategy is l2.strategy
+
     def test_mul(self):
         l1 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         l2 = l1.mul(2)
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3442,6 +3442,29 @@
 
         a.build_types(f, [str])
 
+    def test_negative_number_find(self):
+        def f(s, e):
+            return "xyz".find("x", s, e)
+
+        a = self.RPythonAnnotator()
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        def f(s, e):
+            return "xyz".rfind("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+
+        def f(s, e):
+            return "xyz".count("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        
+
     def test_setslice(self):
         def f():
             lst = [2, 5, 7]
@@ -4080,7 +4103,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify instances with no common base class" 
+        assert ("RPython cannot unify instances with no common base class"
                 in exc.value.msg)
 
     def test_unionerror_iters(self):
@@ -4096,7 +4119,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify incompatible iterator variants" in 
+        assert ("RPython cannot unify incompatible iterator variants" in
                 exc.value.msg)
 
     def test_variable_getattr(self):

From noreply at buildbot.pypy.org  Thu Oct 10 17:50:27 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 10 Oct 2013 17:50:27 +0200 (CEST)
Subject: [pypy-commit] pypy cpyext-best_base: add a passing test: getitem is
 returning a dtype instance that can be used as an int
Message-ID: <20131010155027.2C1281C02EA@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-best_base
Changeset: r67300:45a97e4c2960
Date: 2013-10-10 18:50 +0300
http://bitbucket.org/pypy/pypy/changeset/45a97e4c2960/

Log:	add a passing test: getitem is returning a dtype instance that can
	be used as an int

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -320,6 +320,24 @@
         obj = foo.new()
         assert module.hack_tp_dict(obj) == 2
 
+    def test_getitem(self):
+        import numpypy as np
+        module = self.import_extension('foo', [
+            ("test_getitem", "METH_O",
+             '''
+                PyObject *j, *retval;
+                j = PyInt_FromLong(2);
+                retval = PyObject_GetItem(args, j);
+                Py_DECREF(j);
+                return retval;
+            '''),
+            ],
+            )
+        val = module.test_getitem([10, 11, 12, 13, 14])
+        assert val == 12
+        val = module.test_getitem(np.array([20, 21, 22, 23, 24, 25]))
+        assert val == 22
+
 
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):
@@ -365,6 +383,7 @@
         ref = make_ref(space, w_obj)
         api.Py_DecRef(ref)
 
+
 class AppTestSlots(AppTestCpythonExtensionBase):
     def test_some_slots(self):
         module = self.import_extension('foo', [

From noreply at buildbot.pypy.org  Fri Oct 11 08:16:42 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 08:16:42 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: Cannot use llhelper() in
 case we want to translate the ll_dict_lookup()
Message-ID: <20131011061642.CD7271C0113@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67301:162eedd7917a
Date: 2013-10-11 08:15 +0200
http://bitbucket.org/pypy/pypy/changeset/162eedd7917a/

Log:	Cannot use llhelper() in case we want to translate the
	ll_dict_lookup() function

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -43,7 +43,7 @@
 def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
                 ll_fasthash_function=None, ll_hash_function=None,
                 ll_eq_function=None, method_cache={},
-                dummykeyobj=None, dummyvalueobj=None):
+                dummykeyobj=None, dummyvalueobj=None, rtyper=None):
     # get the actual DICT type. if DICT is None, it's created, otherwise
     # forward reference is becoming DICT
     if DICT is None:
@@ -141,6 +141,11 @@
     adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
 
     family = LookupFamily()
+    adtmeths['lookup_family'] = family
+
+    DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
+                                *fields))
+
     family.empty_array = DICTENTRYARRAY.allocate(0)
     for name, T in [('byte', rffi.UCHAR),
                     ('short', rffi.USHORT),
@@ -149,14 +154,17 @@
         if name == 'int' and not IS_64BIT:
             continue
         lookupfn, storecleanfn = new_lookup_functions(LOOKUP_FUNC,
-                                                      STORECLEAN_FUNC, T=T)
+                                                      STORECLEAN_FUNC, T=T,
+                                                      rtyper=rtyper)
         setattr(family, '%s_lookup_function' % name, lookupfn)
         setattr(family, '%s_insert_clean_function' % name, storecleanfn)
-    adtmeths['lookup_family'] = family
+    return DICT
 
-    DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
-                                *fields))
-    return DICT
+def llhelper_or_compile(rtyper, FUNCPTR, ll_func):
+    if rtyper is None:
+        return llhelper(FUNCPTR, ll_func)
+    else:
+        return rtyper.annotate_helper(ll_func, FUNCPTR.TO.ARGS)
 
 class LookupFamily:
     def _freeze_(self):
@@ -219,7 +227,8 @@
             kwd['dummyvalueobj'] = self.value_repr.get_ll_dummyval_obj(
                 self.rtyper, s_value)
 
-            get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT, **kwd)
+            get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT,
+                        rtyper=self.rtyper, **kwd)
 
     def convert_const(self, dictobj):
         from rpython.rtyper.lltypesystem import llmemory
@@ -251,14 +260,16 @@
                 for dictkeycontainer, dictvalue in dictobj._dict.items():
                     llkey = r_key.convert_const(dictkeycontainer.key)
                     llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_setitem(l_dict, llkey, llvalue)
+                    _ll_dict_insertclean(l_dict, llkey, llvalue,
+                                         dictkeycontainer.hash)
                 return l_dict
 
             else:
                 for dictkey, dictvalue in dictobj.items():
                     llkey = r_key.convert_const(dictkey)
                     llvalue = r_value.convert_const(dictvalue)
-                    ll_dict_setitem(l_dict, llkey, llvalue)
+                    _ll_dict_insertclean(l_dict, llkey, llvalue,
+                                         l_dict.keyhash(llkey))
                 return l_dict
 
     def rtype_len(self, hop):
@@ -530,6 +541,23 @@
             ll_assert(rc > 0, "ll_dict_resize failed?")
         d.resize_counter = rc
 
+def _ll_dict_insertclean(d, key, value, hash):
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    insertcleanfn = ll_pick_insert_clean_function(d)
+    insertcleanfn(d, hash, d.num_used_items)
+    entry = d.entries[d.num_used_items]
+    entry.key = key
+    entry.value = value
+    if hasattr(ENTRY, 'f_hash'):
+        entry.f_hash = hash
+    if hasattr(ENTRY, 'f_valid'):
+        entry.f_valid = True
+    d.num_used_items += 1
+    d.num_items += 1
+    rc = d.resize_counter - 3
+    ll_assert(rc > 0, "_ll_dict_insertclean: overflow")
+    d.resize_counter = rc
+
 def _ll_len_of_d_indexes(d):
     # xxx Haaaack: returns len(d.indexes).  Works independently of
     # the exact type pointed to by d, using a forced cast...
@@ -691,7 +719,7 @@
 FLAG_DELETE = 2
 FLAG_DELETE_TRY_HARD = 3
 
-def new_lookup_functions(LOOKUP_FUNC, STORECLEAN_FUNC, T):
+def new_lookup_functions(LOOKUP_FUNC, STORECLEAN_FUNC, T, rtyper=None):
     INDEXES = lltype.Ptr(lltype.GcArray(T))
 
     def ll_kill_something(d):
@@ -816,8 +844,8 @@
             perturb >>= PERTURB_SHIFT
         indexes[i] = rffi.cast(T, index + VALID_OFFSET)
 
-    return (llhelper(LOOKUP_FUNC, ll_dict_lookup),
-            llhelper(STORECLEAN_FUNC, ll_dict_store_clean))
+    return (llhelper_or_compile(rtyper, LOOKUP_FUNC, ll_dict_lookup),
+            llhelper_or_compile(rtyper, STORECLEAN_FUNC, ll_dict_store_clean))
 
 # ____________________________________________________________
 #
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -856,6 +856,23 @@
         res = self.interpret(f, [5])
         assert res == 25019999
 
+    def test_prebuilt_r_dict(self):
+        def myeq(n, m):
+            return n // 2 == m // 2
+        def myhash(n):
+            return n // 2
+        d = r_dict(myeq, myhash)
+        for i in range(10):
+            d[i] = i*i
+        def f(n):
+            assert len(d) == 5
+            return d[n]
+        assert f(6) == 49
+        res = self.interpret(f, [6])
+        assert res == 49
+        res = self.interpret(f, [5])
+        assert res == 25
+
     def test_resize_during_iteration(self):
         def func():
             d = {5: 1, 6: 2, 7: 3}

From noreply at buildbot.pypy.org  Fri Oct 11 08:50:06 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 08:50:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: Ignore branches
Message-ID: <20131011065006.CF9501C0203@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67302:c952caeecb03
Date: 2013-10-11 08:49 +0200
http://bitbucket.org/pypy/pypy/changeset/c952caeecb03/

Log:	Ignore branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -90,7 +90,8 @@
 .. branch: no-release-gil
 .. branch: safe-win-mmap
 .. branch: boolean-indexing-cleanup
-.. branch: cpyyest-best_base
+.. branch: cpyext-best_base
+.. branch: fileops2
 
 .. branch: nobold-backtrace
 Work on improving UnionError messages and stack trace displays.

From noreply at buildbot.pypy.org  Fri Oct 11 08:57:15 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 08:57:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for test_direct
Message-ID: <20131011065715.166661C0203@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67303:ea8ec1efd690
Date: 2013-10-11 08:56 +0200
http://bitbucket.org/pypy/pypy/changeset/ea8ec1efd690/

Log:	Fix for test_direct

diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1201,7 +1201,6 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
-            assert self.card_page_indices > 0
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
@@ -1236,6 +1235,7 @@
 
     def manually_copy_card_bits(self, source_addr, dest_addr, length):
         # manually copy the individual card marks from source to dest
+        assert self.card_page_indices > 0
         bytes = self.card_marking_bytes_for_length(length)
         #
         anybyte = 0

From noreply at buildbot.pypy.org  Fri Oct 11 09:02:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 09:02:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix tests
Message-ID: <20131011070235.C0FE51C0203@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67304:731bb6f2c1d4
Date: 2013-10-11 09:01 +0200
http://bitbucket.org/pypy/pypy/changeset/731bb6f2c1d4/

Log:	Fix tests

diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -3,6 +3,7 @@
 import py
 
 from rpython.flowspace.model import summary
+from rpython.annotator.model import AnnotatorError
 from rpython.rtyper.lltypesystem.lltype import typeOf, Signed, malloc
 from rpython.rtyper.lltypesystem.rstr import LLHelpers, STR
 from rpython.rtyper.rstr import AbstractLLHelpers
@@ -361,16 +362,16 @@
             res = self.interpret(fn, [i, j])
             assert res == fn(i, j)
 
-    def test_find_TyperError(self):
+    def test_find_AnnotatorError(self):
         const = self.const
         def f():
             s = const('abc')
             s.find(s, 0, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
         def f():
             s = const('abc')
             s.find(s, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
 
     def test_find_empty_string(self):
         const = self.const
@@ -420,9 +421,8 @@
         const = self.const
         def f(i):
             return const("abc").rfind(const(''), i)
-        e = py.test.raises(TyperError, self.interpret, f, [-5])
-        assert str(e.value).startswith(
-            'str.rfind() start must be proven non-negative')
+        e = py.test.raises(AnnotatorError, self.interpret, f, [-5])
+        assert "rfind: not proven to have non-negative start" in str(e.value)
 
     def test_find_char(self):
         const = self.const
@@ -900,16 +900,16 @@
         res = self.interpret(fn, [])
         assert res == 1
 
-    def test_count_TyperError(self):
+    def test_count_AnnotatorError(self):
         const = self.const
         def f():
             s = const('abc')
             s.count(s, 0, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
         def f():
             s = const('abc')
             s.count(s, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
 
     def test_getitem_exc(self):
         const = self.const

From noreply at buildbot.pypy.org  Fri Oct 11 09:51:09 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 11 Oct 2013 09:51:09 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: rpython fix
Message-ID: <20131011075109.DDFC01C0113@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67305:dce6777397d0
Date: 2013-10-11 09:50 +0200
http://bitbucket.org/pypy/pypy/changeset/dce6777397d0/

Log:	rpython fix

diff --git a/rpython/rlib/rarray.py b/rpython/rlib/rarray.py
--- a/rpython/rlib/rarray.py
+++ b/rpython/rlib/rarray.py
@@ -1,6 +1,7 @@
 from rpython.annotator import model as annmodel
 from rpython.annotator.listdef import ListDef
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.pairtype import pair
@@ -48,6 +49,7 @@
 get_raw_buf._always_inline_ = True
 
 
+ at jit.dont_look_inside
 def ll_copy_list_to_raw_array(ll_list, dst_ptr):
     # this code is delicate: we must ensure that there are no GC operations
     # around the call to raw_memcopy
@@ -61,6 +63,7 @@
     # end of no-GC section
 
 
+ at jit.dont_look_inside
 def ll_populate_list_from_raw_array(ll_list, src_ptr, length):
     ITEM = lltype.typeOf(src_ptr).TO.OF
     size = llmemory.sizeof(ITEM) * length

From noreply at buildbot.pypy.org  Fri Oct 11 11:21:41 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 11 Oct 2013 11:21:41 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: remove force_non_null flag,
 not used any more
Message-ID: <20131011092141.6D2181C021C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67306:e4d96795fc24
Date: 2013-10-11 11:20 +0200
http://bitbucket.org/pypy/pypy/changeset/e4d96795fc24/

Log:	remove force_non_null flag, not used any more

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -296,13 +296,12 @@
             listdef.generalize_range_step(flags['range_step'])
         return SomeList(listdef)
 
-    def getdictdef(self, is_r_dict=False, force_non_null=False):
+    def getdictdef(self, is_r_dict=False):
         """Get the DictDef associated with the current position."""
         try:
             dictdef = self.dictdefs[self.position_key]
         except KeyError:
-            dictdef = DictDef(self, is_r_dict=is_r_dict,
-                              force_non_null=force_non_null)
+            dictdef = DictDef(self, is_r_dict=is_r_dict)
             self.dictdefs[self.position_key] = dictdef
         return dictdef
 
diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -287,14 +287,8 @@
             clsdef = clsdef.commonbase(cdef)
     return SomeInstance(clsdef)
 
-def robjmodel_r_dict(s_eqfn, s_hashfn, s_force_non_null=None):
-    if s_force_non_null is None:
-        force_non_null = False
-    else:
-        assert s_force_non_null.is_constant()
-        force_non_null = s_force_non_null.const
-    dictdef = getbookkeeper().getdictdef(is_r_dict=True,
-                                         force_non_null=force_non_null)
+def robjmodel_r_dict(s_eqfn, s_hashfn):
+    dictdef = getbookkeeper().getdictdef(is_r_dict=True)
     dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn)
     return SomeDict(dictdef)
 
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -628,11 +628,10 @@
     The functions key_eq() and key_hash() are used by the key comparison
     algorithm."""
 
-    def __init__(self, key_eq, key_hash, force_non_null=False):
+    def __init__(self, key_eq, key_hash):
         self._dict = {}
         self.key_eq = key_eq
         self.key_hash = key_hash
-        self.force_non_null = force_non_null
 
     def __getitem__(self, key):
         return self._dict[_r_dictkey(self, key)]
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -174,7 +174,7 @@
 class DictRepr(AbstractDictRepr):
 
     def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
-                 custom_eq_hash=None, force_non_null=False):
+                 custom_eq_hash=None):
         self.rtyper = rtyper
         self.DICT = lltype.GcForwardReference()
         self.lowleveltype = lltype.Ptr(self.DICT)
@@ -193,7 +193,6 @@
         self.dictvalue = dictvalue
         self.dict_cache = {}
         self._custom_eq_hash_repr = custom_eq_hash
-        self.force_non_null = force_non_null
         # setup() needs to be called to finish this initialization
 
     def _externalvsinternal(self, rtyper, item_repr):
@@ -212,7 +211,6 @@
             #   able to store dummy values
             s_key   = self.dictkey.s_value
             s_value = self.dictvalue.s_value
-            assert not self.force_non_null # XXX kill the flag
             kwd = {}
             if self.custom_eq_hash:
                 self.r_rdict_eqfn, self.r_rdict_hashfn = (
@@ -886,15 +884,12 @@
     pass
 
 
-def rtype_r_dict(hop, i_force_non_null=None):
+def rtype_r_dict(hop):
     r_dict = hop.r_result
     if not r_dict.custom_eq_hash:
         raise TyperError("r_dict() call does not return an r_dict instance")
     v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
     v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
-    if i_force_non_null is not None:
-        assert i_force_non_null == 2
-        hop.inputarg(lltype.Void, arg=2)
     cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
     hop.exception_cannot_occur()
     v_result = hop.gendirectcall(ll_newdict, cDICT)
diff --git a/rpython/rtyper/rdict.py b/rpython/rtyper/rdict.py
--- a/rpython/rtyper/rdict.py
+++ b/rpython/rtyper/rdict.py
@@ -10,7 +10,6 @@
         dictvalue = self.dictdef.dictvalue
         s_key = dictkey.s_value
         s_value = dictvalue.s_value
-        force_non_null = self.dictdef.force_non_null
         if dictkey.custom_eq_hash:
             custom_eq_hash = lambda: (rtyper.getrepr(dictkey.s_rdict_eqfn),
                                       rtyper.getrepr(dictkey.s_rdict_hashfn))
@@ -18,7 +17,7 @@
             custom_eq_hash = None
         return DictRepr(rtyper, lambda: rtyper.getrepr(s_key),
                         lambda: rtyper.getrepr(s_value), dictkey, dictvalue,
-                        custom_eq_hash, force_non_null)
+                        custom_eq_hash)
 
     def rtyper_makekey(self):
         self.dictdef.dictkey  .dont_change_any_more = True
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1264,25 +1264,6 @@
         res = f()
         assert res == 1
 
-    def test_nonnull_hint(self):
-        def eq(a, b):
-            return a == b
-        def rhash(a):
-            return 3
-
-        def func(i):
-            d = r_dict(eq, rhash, force_non_null=True)
-            if not i:
-                d[None] = i
-            else:
-                d[str(i)] = i
-            return "12" in d, d
-
-        llres = self.interpret(func, [12])
-        assert llres.item0 == 1
-        DICT = lltype.typeOf(llres.item1)
-        assert sorted(DICT.TO.entries.TO.OF._flds) == ['f_hash', 'key', 'value']
-
     def test_memoryerror_should_not_insert(self):
         # This shows a misbehaviour that also exists in CPython 2.7, but not
         # any more in CPython 3.3.  The behaviour is that even if a dict

From noreply at buildbot.pypy.org  Fri Oct 11 11:42:15 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 11 Oct 2013 11:42:15 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: make ll_lookup_function
	rpython
Message-ID: <20131011094215.58AEA1C0370@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67307:3dca24ed5557
Date: 2013-10-11 11:41 +0200
http://bitbucket.org/pypy/pypy/changeset/3dca24ed5557/

Log:	make ll_lookup_function rpython

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -164,7 +164,7 @@
     if rtyper is None:
         return llhelper(FUNCPTR, ll_func)
     else:
-        return rtyper.annotate_helper(ll_func, FUNCPTR.TO.ARGS)
+        return rtyper.annotate_helper_fn(ll_func, FUNCPTR.TO.ARGS)
 
 class LookupFamily:
     def _freeze_(self):
@@ -722,10 +722,11 @@
 
     def ll_kill_something(d):
         i = 0
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
         while True:
-            index = rffi.cast(lltype.Signed, d.indexes[i])
+            index = rffi.cast(lltype.Signed, indexes[i])
             if index >= VALID_OFFSET:
-                d.indexes[i] = rffi.cast(T, DELETED)
+                indexes[i] = rffi.cast(T, DELETED)
                 return index
             i += 1
 
@@ -739,7 +740,7 @@
         # do the first try before any looping
         ENTRIES = lltype.typeOf(entries).TO
         direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
-        index = rffi.cast(lltype.Signed, indexes[i])
+        index = rffi.cast(lltype.Signed, indexes[intmask(i)])
         if index >= VALID_OFFSET:
             checkingkey = entries[index - VALID_OFFSET].key
             if direct_compare and checkingkey == key:
@@ -752,24 +753,18 @@
                 found = d.keyeq(checkingkey, key)
                 #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found)
                 if d.paranoia:
-                    XXX
-                    if (entries != d.entries or indexes != d.indexes or
-                        not entries.valid(ll_index_getitem(d.size, indexes, i))
-                        or entries.getitem_clean(index).key != checkingkey):
+                    if (entries != d.entries or lltype.cast_opaque_ptr(llmemory.GCREF, indexes) != d.indexes or
+                        not entries.valid(index - VALID_OFFSET) or
+                        entries[index - VALID_OFFSET].key != checkingkey):
                         # the compare did major nasty stuff to the dict: start over
-                        if d_signed_indexes(d):
-                            return ll_dict_lookup(d, key, hash,
-                                                  ll_index_getitem_signed)
-                        else:
-                            return ll_dict_lookup(d, key, hash,
-                                                  ll_index_getitem_int)
+                        return ll_dict_lookup(d, key, hash, store_flag)
                 if found:
                     if store_flag == FLAG_DELETE:
                         indexes[i] = rffi.cast(T, DELETED)
                     return index - VALID_OFFSET
             deletedslot = -1
         elif index == DELETED:
-            deletedslot = i
+            deletedslot = intmask(i)
         else:
             # pristine entry -- lookup failed
             if store_flag == FLAG_STORE:
@@ -784,14 +779,12 @@
         while 1:
             # compute the next index using unsigned arithmetic
             i = (i << 2) + i + perturb + 1
-            i = intmask(i) & mask
-            # keep 'i' as a signed number here, to consistently pass signed
-            # arguments to the small helper methods.
-            index = rffi.cast(lltype.Signed, indexes[i])
+            i = i & mask
+            index = rffi.cast(lltype.Signed, indexes[intmask(i)])
             if index == FREE:
                 if store_flag == FLAG_STORE:
                     if deletedslot == -1:
-                        deletedslot = i
+                        deletedslot = intmask(i)
                     indexes[deletedslot] = rffi.cast(T, d.num_used_items +
                                                      VALID_OFFSET)
                 elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
@@ -808,24 +801,17 @@
                     # an equal object
                     found = d.keyeq(checkingkey, key)
                     if d.paranoia:
-                        XXX
-                        if (entries != d.entries or indexes != d.indexes or
-                            not entries.valid(ll_index_getitem(d.size, indexes, i)) or
-                            entries.getitem_clean(index).key != checkingkey):
-                            # the compare did major nasty stuff to the dict:
-                            # start over
-                            if d_signed_indexes(d):
-                                return ll_dict_lookup(d, key, hash,
-                                                      ll_index_getitem_signed)
-                            else:
-                                return ll_dict_lookup(d, key, hash,
-                                                      ll_index_getitem_int)
+                        if (entries != d.entries or lltype.cast_opaque_ptr(llmemory.GCREF, indexes) != d.indexes or
+                            not entries.valid(index - VALID_OFFSET) or
+                            entries[index - VALID_OFFSET].key != checkingkey):
+                            # the compare did major nasty stuff to the dict: start over
+                            return ll_dict_lookup(d, key, hash, store_flag)
                     if found:
                         if store_flag == FLAG_DELETE:
                             indexes[i] = rffi.cast(T, DELETED)
                         return index - VALID_OFFSET
             elif deletedslot == -1:
-                deletedslot = i
+                deletedslot = intmask(i)
             perturb >>= PERTURB_SHIFT
 
     def ll_dict_store_clean(d, hash, index):

From noreply at buildbot.pypy.org  Fri Oct 11 11:45:21 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 11 Oct 2013 11:45:21 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: kill the flag a little
	better
Message-ID: <20131011094521.D15491C0370@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67308:02efe5a1512a
Date: 2013-10-11 11:44 +0200
http://bitbucket.org/pypy/pypy/changeset/02efe5a1512a/

Log:	kill the flag a little better

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -8,7 +8,6 @@
 from pypy.objspace.std.util import negate
 
 from rpython.rlib import jit, rerased
-from rpython.rlib.debug import mark_dict_non_null
 from rpython.rlib.objectmodel import newlist_hint, r_dict, specialize
 from rpython.tool.sourcetools import func_renamer, func_with_new_name
 
@@ -854,8 +853,7 @@
         return True
 
     def get_empty_storage(self):
-        new_dict = r_dict(self.space.eq_w, self.space.hash_w,
-                          force_non_null=True)
+        new_dict = r_dict(self.space.eq_w, self.space.hash_w)
         return self.erase(new_dict)
 
     def _never_equal_to(self, w_lookup_type):
@@ -890,7 +888,6 @@
 
     def get_empty_storage(self):
         res = {}
-        mark_dict_non_null(res)
         return self.erase(res)
 
     def _never_equal_to(self, w_lookup_type):
@@ -954,7 +951,6 @@
 
     def get_empty_storage(self):
         res = {}
-        mark_dict_non_null(res)
         return self.erase(res)
 
     def _never_equal_to(self, w_lookup_type):
diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py
--- a/pypy/objspace/std/identitydict.py
+++ b/pypy/objspace/std/identitydict.py
@@ -2,7 +2,6 @@
 ## dict strategy (see dictmultiobject.py)
 
 from rpython.rlib import rerased
-from rpython.rlib.debug import mark_dict_non_null
 from pypy.objspace.std.dictmultiobject import (AbstractTypedStrategy,
                                                DictStrategy,
                                                create_iterator_classes)
@@ -66,7 +65,6 @@
 
     def get_empty_storage(self):
         d = {}
-        mark_dict_non_null(d)
         return self.erase(d)
 
     def is_correct_type(self, w_obj):
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -1533,7 +1533,7 @@
 # some helper functions
 
 def newset(space):
-    return r_dict(space.eq_w, space.hash_w, force_non_null=True)
+    return r_dict(space.eq_w, space.hash_w)
 
 def set_strategy_and_setdata(space, w_set, w_iterable):
     if w_iterable is None :
diff --git a/rpython/annotator/dictdef.py b/rpython/annotator/dictdef.py
--- a/rpython/annotator/dictdef.py
+++ b/rpython/annotator/dictdef.py
@@ -84,14 +84,12 @@
 
     def __init__(self, bookkeeper, s_key = s_ImpossibleValue,
                                  s_value = s_ImpossibleValue,
-                               is_r_dict = False,
-                           force_non_null = False):
+                                 is_r_dict = False):
         self.dictkey = DictKey(bookkeeper, s_key, is_r_dict)
         self.dictkey.itemof[self] = True
         self.dictvalue = DictValue(bookkeeper, s_value)
         self.dictvalue.itemof[self] = True
         self.bookkeeper = bookkeeper
-        self.force_non_null = force_non_null
 
     def read_key(self, position_key=None):
         if position_key is None:
diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -303,28 +303,6 @@
         return hop.inputarg(hop.args_r[0], arg=0)
 
 
-def mark_dict_non_null(d):
-    """ Mark dictionary as having non-null keys and values. A warning would
-    be emitted (not an error!) in case annotation disagrees.
-    """
-    assert isinstance(d, dict)
-    return d
-
-
-class DictMarkEntry(ExtRegistryEntry):
-    _about_ = mark_dict_non_null
-
-    def compute_result_annotation(self, s_dict):
-        from rpython.annotator.model import SomeDict
-
-        assert isinstance(s_dict, SomeDict)
-        s_dict.dictdef.force_non_null = True
-        return s_dict
-
-    def specialize_call(self, hop):
-        hop.exception_cannot_occur()
-        return hop.inputarg(hop.args_r[0], arg=0)
-
 class IntegerCanBeNegative(Exception):
     pass
 
diff --git a/rpython/rlib/test/test_debug.py b/rpython/rlib/test/test_debug.py
--- a/rpython/rlib/test/test_debug.py
+++ b/rpython/rlib/test/test_debug.py
@@ -3,8 +3,7 @@
 from rpython.rlib.debug import (check_annotation, make_sure_not_resized,
                              debug_print, debug_start, debug_stop,
                              have_debug_prints, debug_offset, debug_flush,
-                             check_nonneg, IntegerCanBeNegative,
-                             mark_dict_non_null)
+                             check_nonneg, IntegerCanBeNegative)
 from rpython.rlib import debug
 from rpython.rtyper.test.test_llinterp import interpret, gengraph
 
@@ -53,15 +52,6 @@
     py.test.raises(ListChangeUnallowed, interpret, f, [],
                    list_comprehension_operations=True)
 
-def test_mark_dict_non_null():
-    def f():
-        d = {"ac": "bx"}
-        mark_dict_non_null(d)
-        return d
-
-    t, typer, graph = gengraph(f, [])
-    assert sorted(graph.returnblock.inputargs[0].concretetype.TO.entries.TO.OF._flds.keys()) == ['key', 'value']
-
 
 class DebugTests(object):
 

From noreply at buildbot.pypy.org  Fri Oct 11 12:02:30 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 11 Oct 2013 12:02:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: implement an obscure flag for dtoa
Message-ID: <20131011100230.8CB4A1C0370@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67309:57e14783523b
Date: 2013-10-11 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/57e14783523b/

Log:	implement an obscure flag for dtoa

diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py
--- a/rpython/rlib/rdtoa.py
+++ b/rpython/rlib/rdtoa.py
@@ -217,13 +217,13 @@
 
         if exp >= 0:
             exp_str = str(exp)
-            if len(exp_str) < 2:
+            if len(exp_str) < 2 and not (flags & rfloat.DTSF_CUT_EXP_0):
                 s += e + '+0' + exp_str
             else:
                 s += e + '+' + exp_str
         else:
             exp_str = str(-exp)
-            if len(exp_str) < 2:
+            if len(exp_str) < 2 and not (flags & rfloat.DTSF_CUT_EXP_0):
                 s += e + '-0' + exp_str
             else:
                 s += e + '-' + exp_str
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -69,6 +69,7 @@
 DTSF_SIGN      = 0x1
 DTSF_ADD_DOT_0 = 0x2
 DTSF_ALT       = 0x4
+DTSF_CUT_EXP_0 = 0x8
 
 DIST_FINITE   = 1
 DIST_NAN      = 2
diff --git a/rpython/rlib/test/test_rdtoa.py b/rpython/rlib/test/test_rdtoa.py
--- a/rpython/rlib/test/test_rdtoa.py
+++ b/rpython/rlib/test/test_rdtoa.py
@@ -29,3 +29,7 @@
 def test_dtoa_precision():
     assert dtoa(1.1, code='f', precision=2) == "1.10"
     assert dtoa(1e12, code='g', precision=12) == "1e+12"
+
+def test_flag_cut_exp_0():
+    assert dtoa(1.1e9, code="g", precision=2, flags=rfloat.DTSF_CUT_EXP_0) == "1.1e+9"
+    assert dtoa(1.1e-9, code="g", precision=2, flags=rfloat.DTSF_CUT_EXP_0) == "1.1e-9"

From noreply at buildbot.pypy.org  Fri Oct 11 12:02:33 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 11 Oct 2013 12:02:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131011100233.73AD01C0370@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67310:d2d8b7ca8251
Date: 2013-10-11 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/d2d8b7ca8251/

Log:	merge

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -10,8 +10,35 @@
 
 import os
 def get_include():
-    head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.join(head, '../include')
+    """
+    Return the directory that contains the NumPy \\*.h header files.
+
+    Extension modules that need to compile against NumPy should use this
+    function to locate the appropriate include directory.
+
+    Notes
+    -----
+    When using ``distutils``, for example in ``setup.py``.
+    ::
+
+        import numpy as np
+        ...
+        Extension('extension_name', ...
+                include_dirs=[np.get_include()])
+        ...
+
+    """
+    import numpy
+    if getattr(numpy, 'show_config', None) is None:
+        # running from numpy source directory
+        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
+        return os.path.join(head, '../include')
+    else:
+        # using installed numpy core headers
+        import numpy.core as core
+        d = os.path.join(os.path.dirname(core.__file__), 'include')
+    return d
+
 
 
 __all__ = ['__version__', 'get_include']
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -52,6 +52,9 @@
 .. branch: ndarray-subtype
 Allow subclassing ndarray, i.e. matrix
 
+.. branch: ndarray-sort
+Implement ndarray in-place sorting (for numeric types, no non-native byte order)
+
 .. branch: pypy-pyarray
 Implement much of numpy's c api in cpyext, allows (slow) access to ndarray
 from c
@@ -87,6 +90,8 @@
 .. branch: no-release-gil
 .. branch: safe-win-mmap
 .. branch: boolean-indexing-cleanup
+.. branch: cpyext-best_base
+.. branch: fileops2
 
 .. branch: nobold-backtrace
 Work on improving UnionError messages and stack trace displays.
@@ -103,3 +108,5 @@
 
 .. branch: file-support-in-rpython
 make open() and friends rpython
+
+
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -99,7 +99,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii'])
+                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
@@ -196,7 +196,7 @@
         assert PyUnicode_GetDefaultEncoding() == 'ascii'
 
 class AppTestCpythonExtensionBase(LeakCheckingTest):
-    
+
     def setup_class(cls):
         cls.space.getbuiltinmodule("cpyext")
         from pypy.module.imp.importing import importhook
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -156,7 +156,7 @@
             def __init__(self):
                 self.foobar = 32
                 super(UnicodeSubclass2, self).__init__()
-        
+
         newobj = UnicodeSubclass2()
         assert newobj.get_val() == 42
         assert newobj.foobar == 32
@@ -358,6 +358,13 @@
         assert w_obj is None
         assert api.PyErr_Occurred() is None
 
+    def test_ndarray_ref(self, space, api):
+        w_obj = space.appexec([], """():
+            import numpypy as np
+            return np.int64(2)""")
+        ref = make_ref(space, w_obj)
+        api.Py_DecRef(ref)
+
 class AppTestSlots(AppTestCpythonExtensionBase):
     def test_some_slots(self):
         module = self.import_extension('foo', [
@@ -525,7 +532,7 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
-        
+
     def test_bool(self):
         module = self.import_extension('foo', [
             ("newInt", "METH_VARARGS",
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -4,7 +4,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.annlowlevel import llhelper
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
-from pypy.objspace.std.typeobject import W_TypeObject
+from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
@@ -574,33 +574,7 @@
 def best_base(space, bases_w):
     if not bases_w:
         return None
-
-    w_winner = None
-    w_base = None
-    for w_base_i in bases_w:
-        if isinstance(w_base_i, W_ClassObject):
-            # old-style base
-            continue
-        assert isinstance(w_base_i, W_TypeObject)
-        w_candidate = solid_base(space, w_base_i)
-        if not w_winner:
-            w_winner = w_candidate
-            w_base = w_base_i
-        elif space.abstract_issubclass_w(w_winner, w_candidate):
-            pass
-        elif space.abstract_issubclass_w(w_candidate, w_winner):
-            w_winner = w_candidate
-            w_base = w_base_i
-        else:
-            raise OperationError(
-                space.w_TypeError,
-                space.wrap("multiple bases have instance lay-out conflict"))
-    if w_base is None:
-        raise OperationError(
-            space.w_TypeError,
-                space.wrap("a new-style class can't have only classic bases"))
-
-    return w_base
+    return find_best_base(space, bases_w)
 
 def inherit_slots(space, pto, w_base):
     # XXX missing: nearly everything
diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -356,6 +356,10 @@
         from pypy.module.micronumpy.arrayimpl.sort import argsort_array
         return argsort_array(self, space, w_axis)
 
+    def sort(self, space, w_axis, w_order):
+        from pypy.module.micronumpy.arrayimpl.sort import sort_array
+        return sort_array(self, space, w_axis, w_order)
+
     def base(self):
         return None
 
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -17,7 +17,7 @@
 
 INT_SIZE = rffi.sizeof(lltype.Signed)
 
-def make_sort_function(space, itemtype, comp_type, count=1):
+def make_argsort_function(space, itemtype, comp_type, count=1):
     TP = itemtype.T
     step = rffi.sizeof(TP)
 
@@ -137,8 +137,8 @@
         else:
             shape = arr.get_shape()
             if axis < 0:
-                axis = len(shape) + axis - 1
-            if axis < 0 or axis > len(shape):
+                axis = len(shape) + axis
+            if axis < 0 or axis >= len(shape):
                 raise OperationError(space.w_IndexError, space.wrap(
                                                     "Wrong axis %d" % axis))
             iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
@@ -162,7 +162,7 @@
     return argsort
 
 def argsort_array(arr, space, w_axis):
-    cache = space.fromcache(SortCache) # that populates SortClasses
+    cache = space.fromcache(ArgSortCache) # that populates ArgSortClasses
     itemtype = arr.dtype.itemtype
     for tp in all_types:
         if isinstance(itemtype, tp[0]):
@@ -178,6 +178,166 @@
 all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
 all_types = unrolling_iterable(all_types)
 
+def make_sort_function(space, itemtype, comp_type, count=1):
+    TP = itemtype.T
+    step = rffi.sizeof(TP)
+
+    class Repr(object):
+        def __init__(self, stride_size, size, values, start):
+            self.stride_size = stride_size
+            self.start = start
+            self.size = size
+            self.values = values
+
+        def getitem(self, item):
+            if count < 2:
+                v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                                    + self.start)
+            else:
+                v = []
+                for i in range(count):
+                    _v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                                    + self.start + step * i)
+                    v.append(_v)
+            if comp_type == 'int':
+                v = intmask(v)
+            elif comp_type == 'float':
+                v = float(v)
+            elif comp_type == 'complex':
+                v = [float(v[0]),float(v[1])]
+            else:
+                raise NotImplementedError('cannot reach')
+            return (v)
+
+        def setitem(self, idx, item):
+            if count < 2:
+                raw_storage_setitem(self.values, idx * self.stride_size +
+                                self.start, rffi.cast(TP, item))
+            else:
+                i = 0
+                for val in item:
+                    raw_storage_setitem(self.values, idx * self.stride_size +
+                                self.start + i*step, rffi.cast(TP, val))
+                    i += 1
+
+    class ArgArrayRepWithStorage(Repr):
+        def __init__(self, stride_size, size):
+            start = 0
+            values = alloc_raw_storage(size * stride_size,
+                                            track_allocation=False)
+            Repr.__init__(self, stride_size,
+                          size, values, start)
+
+        def __del__(self):
+            free_raw_storage(self.values, track_allocation=False)
+
+    def arg_getitem(lst, item):
+        return lst.getitem(item)
+
+    def arg_setitem(lst, item, value):
+        lst.setitem(item, value)
+
+    def arg_length(lst):
+        return lst.size
+
+    def arg_getitem_slice(lst, start, stop):
+        retval = ArgArrayRepWithStorage(lst.stride_size, stop-start)
+        for i in range(stop-start):
+            retval.setitem(i, lst.getitem(i+start))
+        return retval
+
+    if count < 2:
+        def arg_lt(a, b):
+            # handles NAN and INF
+            return a < b or b != b and a == a
+    else:
+        def arg_lt(a, b):
+            for i in range(count):
+                if b[i] != b[i] and a[i] == a[i]:
+                    return True
+                elif b[i] == b[i] and a[i] != a[i]:
+                    return False
+            for i in range(count):
+                if a[i] < b[i]:
+                    return True
+                elif a[i] > b[i]:
+                    return False
+            # Does numpy do True?
+            return False
+
+    ArgSort = make_timsort_class(arg_getitem, arg_setitem, arg_length,
+                                 arg_getitem_slice, arg_lt)
+
+    def sort(arr, space, w_axis, itemsize):
+        if w_axis is space.w_None:
+            # note that it's fine to pass None here as we're not going
+            # to pass the result around (None is the link to base in slices)
+            arr = arr.reshape(space, None, [arr.get_size()])
+            axis = 0
+        elif w_axis is None:
+            axis = -1
+        else:
+            axis = space.int_w(w_axis)
+        # create array of indexes
+        if len(arr.get_shape()) == 1:
+            r = Repr(itemsize, arr.get_size(), arr.get_storage(),
+                     arr.start)
+            ArgSort(r).sort()
+        else:
+            shape = arr.get_shape()
+            if axis < 0:
+                axis = len(shape) + axis
+            if axis < 0 or axis >= len(shape):
+                raise OperationError(space.w_IndexError, space.wrap(
+                                                    "Wrong axis %d" % axis))
+            iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
+            iter = AxisIterator(arr, iterable_shape, axis, False)
+            stride_size = arr.strides[axis]
+            axis_size = arr.shape[axis]
+            while not iter.done():
+                r = Repr(stride_size, axis_size, arr.get_storage(), iter.offset)
+                ArgSort(r).sort()
+                iter.next()
+
+    return sort
+
+def sort_array(arr, space, w_axis, w_order):
+    cache = space.fromcache(SortCache) # that populates SortClasses
+    itemtype = arr.dtype.itemtype
+    if not arr.dtype.native:
+        raise OperationError(space.w_NotImplementedError,
+           space.wrap("sorting of non-native btyeorder not supported yet"))
+    for tp in all_types:
+        if isinstance(itemtype, tp[0]):
+            return cache._lookup(tp)(arr, space, w_axis,
+                                     itemtype.get_element_size())
+    # XXX this should probably be changed
+    raise OperationError(space.w_NotImplementedError,
+           space.wrap("sorting of non-numeric types " + \
+                  "'%s' is not implemented" % arr.dtype.get_name(), ))
+
+all_types = (types.all_float_types + types.all_complex_types +
+             types.all_int_types)
+all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
+all_types = unrolling_iterable(all_types)
+
+class ArgSortCache(object):
+    built = False
+
+    def __init__(self, space):
+        if self.built:
+            return
+        self.built = True
+        cache = {}
+        for cls, it in all_types._items:
+            if it == 'complex':
+                cache[cls] = make_argsort_function(space, cls, it, 2)
+            else:
+                cache[cls] = make_argsort_function(space, cls, it)
+        self.cache = cache
+        self._lookup = specialize.memo()(lambda tp : cache[tp[0]])
+
+
 class SortCache(object):
     built = False
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -629,9 +629,13 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "setflags not implemented yet"))
 
-    def descr_sort(self, space, w_axis=-1, w_kind='quicksort', w_order=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "sort not implemented yet"))
+    @unwrap_spec(kind=str)
+    def descr_sort(self, space, w_axis=None, kind='quicksort', w_order=None):
+        # happily ignore the kind
+        # modify the array in-place
+        if self.is_scalar():
+            return
+        return self.implementation.sort(space, w_axis, w_order)
 
     def descr_squeeze(self, space):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -1118,6 +1122,7 @@
     conj = interp2app(W_NDimArray.descr_conj),
 
     argsort  = interp2app(W_NDimArray.descr_argsort),
+    sort  = interp2app(W_NDimArray.descr_sort),
     astype   = interp2app(W_NDimArray.descr_astype),
     base     = GetSetProperty(W_NDimArray.descr_get_base),
     byteswap = interp2app(W_NDimArray.descr_byteswap),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2652,55 +2652,6 @@
         assert array([1, 2, 3], '>i2')[::2].tostring() == '\x00\x01\x00\x03'
         assert array(0, dtype='i2').tostring() == '\x00\x00'
 
-    def test_argsort_dtypes(self):
-        from numpypy import array, arange
-        assert array(2.0).argsort() == 0
-        nnp = self.non_native_prefix
-        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
-                        nnp + 'i2', complex]:
-            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
-            c = a.copy()
-            res = a.argsort()
-            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
-                'a,res,dtype %r,%r,%r' % (a,res,dtype)
-            assert (a == c).all() # not modified
-            a = arange(100)
-            assert (a.argsort() == a).all()
-        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
-
-    def test_argsort_nd(self):
-        from numpypy import array
-        a = array([[4, 2], [1, 3]])
-        assert (a.argsort() == [[1, 0], [0, 1]]).all()
-        a = array(range(10) + range(10) + range(10))
-        b = a.argsort()
-        assert (b[:3] == [0, 10, 20]).all()
-        #trigger timsort 'run' mode which calls arg_getitem_slice
-        a = array(range(100) + range(100) + range(100))
-        b = a.argsort()
-        assert (b[:3] == [0, 100, 200]).all()
-        a = array([[[]]]).reshape(3,4,0)
-        b = a.argsort()
-        assert b.size == 0
-
-    def test_argsort_random(self):
-        from numpypy import array
-        from _random import Random
-        rnd = Random(1)
-        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
-        a.argsort()
-
-    def test_argsort_axis(self):
-        from numpypy import array
-        a = array([[4, 2], [1, 3]])
-        assert (a.argsort(axis=None) == [2, 1, 3, 0]).all()
-        assert (a.argsort(axis=-1) == [[1, 0], [0, 1]]).all()
-        assert (a.argsort(axis=0) == [[1, 0], [0, 1]]).all()
-        assert (a.argsort(axis=1) == [[1, 0], [0, 1]]).all()
-        a = array([[3, 2, 1], [1, 2, 3]])
-        assert (a.argsort(axis=0) == [[1, 0, 0], [0, 1, 1]]).all()
-        assert (a.argsort(axis=1) == [[2, 1, 0], [0, 1, 2]]).all()
-
 
 class AppTestRanges(BaseNumpyAppTest):
     def test_arange(self):
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -0,0 +1,322 @@
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestSupport(BaseNumpyAppTest):
+    def setup_class(cls):
+        import struct
+        BaseNumpyAppTest.setup_class.im_func(cls)
+        cls.w_data = cls.space.wrap(struct.pack('dddd', 1, 2, 3, 4))
+        cls.w_fdata = cls.space.wrap(struct.pack('f', 2.3))
+        cls.w_float16val = cls.space.wrap('\x00E') # 5.0 in float16
+        cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
+        cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
+        cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+
+    def test_argsort_dtypes(self):
+        from numpypy import array, arange
+        assert array(2.0).argsort() == 0
+        nnp = self.non_native_prefix
+        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
+                        nnp + 'i2', complex]:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            c = a.copy()
+            res = a.argsort()
+            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
+                'a,res,dtype %r,%r,%r' % (a,res,dtype)
+            assert (a == c).all() # not modified
+            a = arange(100)
+            assert (a.argsort() == a).all()
+        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
+
+    def test_argsort_nd(self):
+        from numpypy import array
+        a = array([[4, 2], [1, 3]])
+        assert (a.argsort() == [[1, 0], [0, 1]]).all()
+        a = array(range(10) + range(10) + range(10))
+        b = a.argsort()
+        assert (b[:3] == [0, 10, 20]).all()
+        #trigger timsort 'run' mode which calls arg_getitem_slice
+        a = array(range(100) + range(100) + range(100))
+        b = a.argsort()
+        assert (b[:3] == [0, 100, 200]).all()
+        a = array([[[]]]).reshape(3,4,0)
+        b = a.argsort()
+        assert b.size == 0
+
+    def test_argsort_random(self):
+        from numpypy import array
+        from _random import Random
+        rnd = Random(1)
+        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
+        a.argsort()
+
+    def test_argsort_axis(self):
+        from numpypy import array
+        a = array([[4, 2], [1, 3]])
+        assert (a.argsort(axis=None) == [2, 1, 3, 0]).all()
+        assert (a.argsort(axis=-1) == [[1, 0], [0, 1]]).all()
+        assert (a.argsort(axis=0) == [[1, 0], [0, 1]]).all()
+        assert (a.argsort(axis=1) == [[1, 0], [0, 1]]).all()
+        a = array([[3, 2, 1], [1, 2, 3]])
+        assert (a.argsort(axis=0) == [[1, 0, 0], [0, 1, 1]]).all()
+        assert (a.argsort(axis=1) == [[2, 1, 0], [0, 1, 2]]).all()
+
+    def test_sort_dtypes(self):
+        from numpypy import array, arange
+        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
+                        'i2', complex]:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
+            c = a.copy()
+            a.sort()
+            assert (a == b).all(), \
+                'a,orig,dtype %r,%r,%r' % (a,c,dtype)
+        a = arange(100)
+        c = a.copy()
+        a.sort()
+        assert (a == c).all()
+
+    def test_sort_dtypesi_nonnative(self):
+        from numpypy import array
+        nnp = self.non_native_prefix
+        for dtype in [ nnp + 'i2']:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
+            c = a.copy()
+            exc = raises(NotImplementedError, a.sort)
+            assert exc.value[0].find('supported') >= 0
+            #assert (a == b).all(), \
+            #    'a,orig,dtype %r,%r,%r' % (a,c,dtype)
+
+
+# tests from numpy/tests/test_multiarray.py
+    def test_sort_corner_cases(self):
+        # test ordering for floats and complex containing nans. It is only
+        # necessary to check the lessthan comparison, so sorts that
+        # only follow the insertion sort path are sufficient. We only
+        # test doubles and complex doubles as the logic is the same.
+
+        # check doubles
+        from numpypy import array, nan, zeros, complex128, arange
+        from numpy import isnan
+        a = array([nan, 1, 0])
+        b = a.copy()
+        b.sort()
+        assert (isnan(b) == isnan(a[::-1])).all()
+        assert (b[:2] == a[::-1][:2]).all()
+
+        # check complex
+        a = zeros(9, dtype=complex128)
+        a.real += [nan, nan, nan, 1, 0, 1, 1, 0, 0]
+        a.imag += [nan, 1, 0, nan, nan, 1, 0, 1, 0]
+        b = a.copy()
+        b.sort()
+        assert (isnan(b) == isnan(a[::-1])).all()
+        assert (b[:4] == a[::-1][:4]).all()
+
+        # all c scalar sorts use the same code with different types
+        # so it suffices to run a quick check with one type. The number
+        # of sorted items must be greater than ~50 to check the actual
+        # algorithm because quick and merge sort fall over to insertion
+        # sort for small arrays.
+        a = arange(101)
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h'] :
+            msg = "scalar sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+        # test complex sorts. These use the same code as the scalars
+        # but the compare fuction differs.
+        ai = a*1j + 1
+        bi = b*1j + 1
+        for kind in ['q', 'm', 'h'] :
+            msg = "complex sort, real part == 1, kind=%s" % kind
+            c = ai.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+            c = bi.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+        ai = a + 1j
+        bi = b + 1j
+        for kind in ['q', 'm', 'h'] :
+            msg = "complex sort, imag part == 1, kind=%s" % kind
+            c = ai.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+            c = bi.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+
+        # check axis handling. This should be the same for all type
+        # specific sorts, so we only check it for one type and one kind
+        a = array([[3, 2], [1, 0]])
+        b = array([[1, 0], [3, 2]])
+        c = array([[2, 3], [0, 1]])
+        d = a.copy()
+        d.sort(axis=0)
+        assert (d == b).all(), "test sort with axis=0"
+        d = a.copy()
+        d.sort(axis=1)
+        assert (d == c).all(), "test sort with axis=1"
+        d = a.copy()
+        d.sort()
+        assert (d == c).all(), "test sort with default axis"
+
+    def test_sort_corner_cases_string_records(self):
+        skip('not implemented yet')
+        from numpypy import array, dtype
+        # test string sorts.
+        s = 'aaaaaaaa'
+        a = array([s + chr(i) for i in range(101)])
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h'] :
+            msg = "string sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+
+        # test record array sorts.
+        dt =dtype([('f', float), ('i', int)])
+        a = array([(i, i) for i in range(101)], dtype = dt)
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "object sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_unicode(self):
+        from numpypy import array
+        # test unicode sorts.
+        s = 'aaaaaaaa'
+        try:
+            a = array([s + chr(i) for i in range(101)], dtype=unicode)
+            b = a[::-1].copy()
+        except:
+            skip('unicode type not supported yet')
+        for kind in ['q', 'm', 'h'] :
+            msg = "unicode sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_objects(self):
+        # test object array sorts.
+        from numpypy import empty
+        try:
+            a = empty((101,), dtype=object)
+        except:
+            skip('object type not supported yet')
+        a[:] = list(range(101))
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "object sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_datetime(self):
+        from numpypy import arange
+        # test datetime64 sorts.
+        try:
+            a = arange(0, 101, dtype='datetime64[D]')
+        except:
+            skip('datetime type not supported yet')
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "datetime64 sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+        # test timedelta64 sorts.
+        a = arange(0, 101, dtype='timedelta64[D]')
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "timedelta64 sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_order(self):
+        from numpypy import array, zeros
+        from sys import byteorder
+        # Test sorting an array with fields
+        skip('not implemented yet')
+        x1 = array([21, 32, 14])
+        x2 = array(['my', 'first', 'name'])
+        x3=array([3.1, 4.5, 6.2])
+        r=zeros(3, dtype=[('id','i'),('word','S5'),('number','f')])
+        r['id'] = x1
+        r['word'] = x2
+        r['number'] = x3
+
+        r.sort(order=['id'])
+        assert (r['id'] == [14, 21, 32]).all()
+        assert (r['word'] == ['name', 'my', 'first']).all()
+        assert max(abs(r['number'] - [6.2, 3.1, 4.5])) < 1e-6
+
+        r.sort(order=['word'])
+        assert (r['id'] == [32, 21, 14]).all()
+        assert (r['word'] == ['first', 'my', 'name']).all()
+        assert max(abs(r['number'] - [4.5, 3.1, 6.2])) < 1e-6
+
+        r.sort(order=['number'])
+        assert (r['id'] == [21, 32, 14]).all()
+        assert (r['word'] == ['my', 'first', 'name']).all()
+        assert max(abs(r['number'] - [3.1, 4.5, 6.2])) < 1e-6
+
+        if byteorder == 'little':
+            strtype = '>i2'
+        else:
+            strtype = '<i2'
+        mydtype = [('name', 'S5'), ('col2', strtype)]
+        r = array([('a', 1), ('b', 255), ('c', 3), ('d', 258)],
+                     dtype= mydtype)
+        r.sort(order='col2')
+        assert (r['col2'] == [1, 3, 255, 258]).all()
+        assert (r == array([('a', 1), ('c', 3), ('b', 255), ('d', 258)],
+                                 dtype=mydtype)).all()
+
+
+# tests from numpy/tests/test_regression.py
+    def test_sort_bigendian(self):
+        skip('not implemented yet')
+        from numpypy import array, dtype
+        a = array(range(11),dtype='float64')
+        c = a.astype(dtype('<f8'))
+        c.sort()
+        assert max(abs(a-c)) < 1e-32
+
+    def test_string_sort_with_zeros(self):
+        skip('not implemented yet')
+        from numpypy import fromstring
+        """Check sort for strings containing zeros."""
+        x = fromstring("\x00\x02\x00\x01", dtype="S2")
+        y = fromstring("\x00\x01\x00\x02", dtype="S2")
+        x.sort(kind='q')
+        assert (x == y).all()
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1864,7 +1864,6 @@
         return interp_boxes.W_StringBox(arr,  0, arr.dtype)
 
     def fill(self, storage, width, box, start, stop, offset):
-        from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArrayNotOwning
         for i in xrange(start, stop, width):
             self._store(storage, i, offset, box)
 
@@ -1873,6 +1872,13 @@
 class UnicodeType(BaseType, BaseStringType):
     T = lltype.UniChar
 
+    @jit.unroll_safe
+    def coerce(self, space, dtype, w_item):
+        if isinstance(w_item, interp_boxes.W_UnicodeBox):
+            return w_item
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "coerce (probably from set_item) not implemented for unicode type"))
+
 NonNativeUnicodeType = UnicodeType
 
 class VoidType(BaseType, BaseStringType):
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -1,5 +1,5 @@
 import operator
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.objspace.std import model, newformat
 from pypy.objspace.std.floattype import float_typedef, W_AbstractFloatObject
 from pypy.objspace.std.multimethod import FailedToImplementArgs
@@ -424,7 +424,16 @@
     x = w_float1.floatval
     y = w_float2.floatval
 
-    return W_FloatObject(_pow(space, x, y))
+    try:
+        result = _pow(space, x, y)
+    except PowDomainError:
+        raise operationerrfmt(space.w_ValueError,
+                              "negative number cannot be raised to a "
+                              "fractional power")
+    return W_FloatObject(result)
+
+class PowDomainError(ValueError):
+    """Signals a negative number raised to a fractional power"""
 
 def _pow(space, x, y):
     # Sort out special cases here instead of relying on pow()
@@ -478,16 +487,14 @@
                                             "a negative power"))
 
     negate_result = False
-    # special case: "(-1.0) ** bignum" should not raise ValueError,
+    # special case: "(-1.0) ** bignum" should not raise PowDomainError,
     # unlike "math.pow(-1.0, bignum)".  See http://mail.python.org/
     # -           pipermail/python-bugs-list/2003-March/016795.html
     if x < 0.0:
         if isnan(y):
             return NAN
         if math.floor(y) != y:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("negative number cannot be "
-                                            "raised to a fractional power"))
+            raise PowDomainError
         # y is an exact integer, albeit perhaps a very large one.
         # Replace x by its absolute value and remember to negate the
         # pow result if y is odd.
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1201,7 +1201,6 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
-            assert self.card_page_indices > 0
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
@@ -1236,6 +1235,7 @@
 
     def manually_copy_card_bits(self, source_addr, dest_addr, length):
         # manually copy the individual card marks from source to dest
+        assert self.card_page_indices > 0
         bytes = self.card_marking_bytes_for_length(length)
         #
         anybyte = 0
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -820,8 +820,9 @@
             entry = entries[i]
             hash = entries.hash(i)
             key = entry.key
+            value = entry.value
             j = ll_dict_lookup(dic1, key, hash)
-            _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j)
+            _ll_dict_setitem_lookup_done(dic1, key, value, hash, j)
         i += 1
 ll_update.oopspec = 'dict.update(dic1, dic2)'
 
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -3,6 +3,7 @@
 import py
 
 from rpython.flowspace.model import summary
+from rpython.annotator.model import AnnotatorError
 from rpython.rtyper.lltypesystem.lltype import typeOf, Signed, malloc
 from rpython.rtyper.lltypesystem.rstr import LLHelpers, STR
 from rpython.rtyper.rstr import AbstractLLHelpers
@@ -361,16 +362,16 @@
             res = self.interpret(fn, [i, j])
             assert res == fn(i, j)
 
-    def test_find_TyperError(self):
+    def test_find_AnnotatorError(self):
         const = self.const
         def f():
             s = const('abc')
             s.find(s, 0, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
         def f():
             s = const('abc')
             s.find(s, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
 
     def test_find_empty_string(self):
         const = self.const
@@ -420,9 +421,8 @@
         const = self.const
         def f(i):
             return const("abc").rfind(const(''), i)
-        e = py.test.raises(TyperError, self.interpret, f, [-5])
-        assert str(e.value).startswith(
-            'str.rfind() start must be proven non-negative')
+        e = py.test.raises(AnnotatorError, self.interpret, f, [-5])
+        assert "rfind: not proven to have non-negative start" in str(e.value)
 
     def test_find_char(self):
         const = self.const
@@ -900,16 +900,16 @@
         res = self.interpret(fn, [])
         assert res == 1
 
-    def test_count_TyperError(self):
+    def test_count_AnnotatorError(self):
         const = self.const
         def f():
             s = const('abc')
             s.count(s, 0, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
         def f():
             s = const('abc')
             s.count(s, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
 
     def test_getitem_exc(self):
         const = self.const

From noreply at buildbot.pypy.org  Fri Oct 11 12:11:36 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 11 Oct 2013 12:11:36 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: we can end up with -1 here,
	but it's ok
Message-ID: <20131011101136.0F75A1C0113@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67311:50d00c894de1
Date: 2013-10-11 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/50d00c894de1/

Log:	we can end up with -1 here, but it's ok

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -553,7 +553,6 @@
     d.num_used_items += 1
     d.num_items += 1
     rc = d.resize_counter - 3
-    ll_assert(rc > 0, "_ll_dict_insertclean: overflow")
     d.resize_counter = rc
 
 def _ll_len_of_d_indexes(d):

From noreply at buildbot.pypy.org  Fri Oct 11 12:14:46 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 11 Oct 2013 12:14:46 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: make the test work by using
 llhelper on pseudortyper
Message-ID: <20131011101446.D3DA41C0203@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67312:29455e90f93a
Date: 2013-10-11 12:14 +0200
http://bitbucket.org/pypy/pypy/changeset/29455e90f93a/

Log:	make the test work by using llhelper on pseudortyper

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -161,7 +161,8 @@
     return DICT
 
 def llhelper_or_compile(rtyper, FUNCPTR, ll_func):
-    if rtyper is None:
+    # the check is for pseudo rtyper from tests
+    if rtyper is None or not hasattr(rtyper, 'annotate_helper_fn'):
         return llhelper(FUNCPTR, ll_func)
     else:
         return rtyper.annotate_helper_fn(ll_func, FUNCPTR.TO.ARGS)

From noreply at buildbot.pypy.org  Fri Oct 11 15:02:41 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 11 Oct 2013 15:02:41 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: this is a better name
Message-ID: <20131011130241.97E0B1D22C5@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67315:e49ce2667f52
Date: 2013-10-11 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/e49ce2667f52/

Log:	this is a better name

diff --git a/pypy/module/_cffi_backend/test/test_extra.py b/pypy/module/_cffi_backend/test/test_fastpath.py
rename from pypy/module/_cffi_backend/test/test_extra.py
rename to pypy/module/_cffi_backend/test/test_fastpath.py

From noreply at buildbot.pypy.org  Fri Oct 11 15:02:38 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 11 Oct 2013 15:02:38 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: move these two methods to
 the very base class
Message-ID: <20131011130238.AE8EB1D22C2@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67313:5019ee2fb1c2
Date: 2013-10-11 14:26 +0200
http://bitbucket.org/pypy/pypy/changeset/5019ee2fb1c2/

Log:	move these two methods to the very base class

diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -43,6 +43,12 @@
     def is_unichar_ptr_or_array(self):
         return False
 
+    def is_long(self):
+        return False
+
+    def is_double(self):
+        return False
+
     def newp(self, w_init):
         space = self.space
         raise operationerrfmt(space.w_TypeError,
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -85,12 +85,6 @@
             return self.space.wrap(s)
         return W_CType.string(self, cdataobj, maxlen)
 
-    def is_long(self):
-        return False
-
-    def is_double(self):
-        return False
-
 class W_CTypePrimitiveCharOrUniChar(W_CTypePrimitive):
     _attrs_ = []
     is_primitive_integer = True

From noreply at buildbot.pypy.org  Fri Oct 11 15:02:39 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 11 Oct 2013 15:02:39 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: fix these test with -A
Message-ID: <20131011130239.EC7F71D22C4@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67314:0654986f3d75
Date: 2013-10-11 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/0654986f3d75/

Log:	fix these test with -A

diff --git a/pypy/module/_cffi_backend/test/test_extra.py b/pypy/module/_cffi_backend/test/test_extra.py
--- a/pypy/module/_cffi_backend/test/test_extra.py
+++ b/pypy/module/_cffi_backend/test/test_extra.py
@@ -53,6 +53,8 @@
             return original(*args)
         self._original = original
         rarray.populate_list_from_raw_array = populate_list_from_raw_array
+        #
+        self.w_runappdirect = self.space.wrap(self.runappdirect)
 
 
     def teardown_method(self, meth):
@@ -70,7 +72,8 @@
         buf[2] = 3
         lst = list(buf)
         assert lst == [1, 2, 3]
-        assert self.get_count() == 1
+        if not self.runappdirect:
+            assert self.get_count() == 1
 
     def test_list_float(self):
         import _cffi_backend
@@ -83,4 +86,5 @@
         buf[2] = 3.3
         lst = list(buf)
         assert lst == [1.1, 2.2, 3.3]
-        assert self.get_count() == 1
+        if not self.runappdirect:
+            assert self.get_count() == 1

From noreply at buildbot.pypy.org  Fri Oct 11 15:02:42 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 11 Oct 2013 15:02:42 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: merge heads
Message-ID: <20131011130242.BBC741D22C2@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67316:4a5a1fe77f06
Date: 2013-10-11 15:02 +0200
http://bitbucket.org/pypy/pypy/changeset/4a5a1fe77f06/

Log:	merge heads

diff --git a/rpython/rlib/rarray.py b/rpython/rlib/rarray.py
--- a/rpython/rlib/rarray.py
+++ b/rpython/rlib/rarray.py
@@ -1,6 +1,7 @@
 from rpython.annotator import model as annmodel
 from rpython.annotator.listdef import ListDef
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.pairtype import pair
@@ -48,6 +49,7 @@
 get_raw_buf._always_inline_ = True
 
 
+ at jit.dont_look_inside
 def ll_copy_list_to_raw_array(ll_list, dst_ptr):
     # this code is delicate: we must ensure that there are no GC operations
     # around the call to raw_memcopy
@@ -61,6 +63,7 @@
     # end of no-GC section
 
 
+ at jit.dont_look_inside
 def ll_populate_list_from_raw_array(ll_list, src_ptr, length):
     ITEM = lltype.typeOf(src_ptr).TO.OF
     size = llmemory.sizeof(ITEM) * length

From noreply at buildbot.pypy.org  Fri Oct 11 16:16:27 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 16:16:27 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: in-progress: various rewrites
	and refactorings
Message-ID: <20131011141627.6F6591C3654@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67317:7a425a44799e
Date: 2013-10-11 16:15 +0200
http://bitbucket.org/pypy/pypy/changeset/7a425a44799e/

Log:	in-progress: various rewrites and refactorings

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -85,7 +85,7 @@
 # on young objects (unless they are large arrays, see below), and we
 # simply assume that any young object can point to any other young object.
 # For old and prebuilt objects, the flag is usually set, and is cleared
-# when we write a young pointer to it.  For large arrays with
+# when we write any pointer to it.  For large arrays with
 # GCFLAG_HAS_CARDS, we rely on card marking to track where the
 # young pointers are; the flag GCFLAG_TRACK_YOUNG_PTRS is set in this
 # case too, to speed up the write barrier.
@@ -102,7 +102,6 @@
 # and on surviving raw-malloced young objects during a minor collection.
 GCFLAG_VISITED      = first_gcflag << 2
 
-
 # The following flag is set on nursery objects of which we asked the id
 # or the identityhash.  It means that a space of the size of the object
 # has already been allocated in the nonmovable part.  The same flag is
@@ -128,26 +127,10 @@
 # This flag is used by the tri color algorithm. An object which
 # has the gray bit set has been marked reachable, but not yet walked
 # by the incremental collection
-GCFLAG_GRAY = first_gcflag << 8
+GCFLAG_GRAY         = first_gcflag << 8
 
-# This flag allows sweeping to be incrementalised.
-# it is set when an object would be swept, but isnt
-# because this flag was not set. The point of this
-# flag is to make sure an object has survived through
-# at least one major collection so we are sure
-# it is unreachable. It is needed because a write
-# barrier has no way of knowing which objects are truly
-# unvisited, or they were simply already reset by
-# a sweep.
-GCFLAG_CANSWEEP      = first_gcflag << 9
+_GCFLAG_FIRST_UNUSED = first_gcflag << 9    # the first unused bit
 
-# Flag indicates object is old. It is needed by the
-# write barrier code so that we can track when a young
-# reference is written into a black object.
-# we must make a shadow and prevent such an object from being freed by
-# the next minor collection so that we dont get dead objects in
-# objects_to_trace during marking.
-GCFLAG_OLD      = first_gcflag << 10
 
 # States for the incremental GC
 
@@ -173,9 +156,6 @@
              'FINALIZING']
 
 
-TID_MASK            = (first_gcflag << 11) - 1
-
-
 FORWARDSTUB = lltype.GcStruct('forwarding_stub',
                               ('forw', llmemory.Address))
 FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB)
@@ -797,7 +777,6 @@
             # is for large objects, bigger than the 'large_objects' threshold,
             # which are raw-malloced but still young.
             extra_flags = GCFLAG_TRACK_YOUNG_PTRS
-            can_make_young = False
             #
         else:
             # No, so proceed to allocate it externally with raw_malloc().
@@ -869,13 +848,7 @@
         if self.is_varsize(typeid):
             offset_to_length = self.varsize_offset_to_length(typeid)
             (result + size_gc_header + offset_to_length).signed[0] = length
-        newobj = result + size_gc_header
-        #
-        # If we are in STATE_MARKING, then the new object must be made gray.
-        if not can_make_young and self.gc_state == STATE_MARKING:
-            self.write_to_visited_object_backward(newobj)
-        #
-        return newobj
+        return result + size_gc_header
 
 
     # ----------
@@ -1019,7 +992,7 @@
             ll_assert(tid == -42, "bogus header for young obj")
         else:
             ll_assert(bool(tid), "bogus header (1)")
-            ll_assert(tid & ~TID_MASK == 0, "bogus header (2)")
+            ll_assert(tid & -_GCFLAG_FIRST_UNUSED == 0, "bogus header (2)")
         return result
 
     def get_forwarding_address(self, obj):
@@ -1112,6 +1085,11 @@
         if self.header(obj).tid & GCFLAG_VISITED != 0:
             # Visited, should NEVER point to a white object.
             self.trace(obj,self._debug_check_not_white,None)
+            # During marking, all visited (black) objects should always have
+            # the GCFLAG_TRACK_YOUNG_PTRS flag set, for the write barrier to
+            # trigger
+            ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS != 0,
+                      "black object without GCFLAG_TRACK_YOUNG_PTRS")
 
         if self.header(obj).tid & GCFLAG_GRAY != 0:
             ll_assert(self._debug_objects_to_trace_dict.contains(obj),
@@ -1193,7 +1171,7 @@
     # for the JIT: a minimal description of the write_barrier() method
     # (the JIT assumes it is of the shape
     #  "if addr_struct.int0 & JIT_WB_IF_FLAG: remember_young_pointer()")
-    JIT_WB_IF_FLAG = GCFLAG_TRACK_YOUNG_PTRS | GCFLAG_VISITED
+    JIT_WB_IF_FLAG = GCFLAG_TRACK_YOUNG_PTRS
 
     # for the JIT to generate custom code corresponding to the array
     # write barrier for the simplest case of cards.  If JIT_CARDS_SET
@@ -1218,13 +1196,11 @@
         return cls.minimal_size_in_nursery
 
     def write_barrier(self, addr_struct):
-        if self.header(addr_struct).tid & (GCFLAG_TRACK_YOUNG_PTRS
-                                           | GCFLAG_VISITED):
+        if self.header(addr_struct).tid & GCFLAG_TRACK_YOUNG_PTRS:
             self.remember_young_pointer(addr_struct)
 
     def write_barrier_from_array(self, addr_array, index):
-        if self.header(addr_array).tid & (GCFLAG_TRACK_YOUNG_PTRS |
-                                          GCFLAG_VISITED):
+        if self.header(addr_array).tid & GCFLAG_TRACK_YOUNG_PTRS:
             self.remember_young_pointer(addr_array)
 
     def write_to_visited_object_backward(self, addr_struct):
@@ -1257,14 +1233,6 @@
                           self.header(addr_struct).tid & GCFLAG_HAS_CARDS != 0,
                       "young object with GCFLAG_TRACK_YOUNG_PTRS and no cards")
             #
-            # This is the write barrier of incremental GC
-            tid = self.header(addr_struct).tid
-            if tid & GCFLAG_VISITED:
-                if self.gc_state == STATE_MARKING:
-                    self.write_to_visited_object_backward(addr_struct)
-                if tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
-                    return    # done
-            #
             # We need to remove the flag GCFLAG_TRACK_YOUNG_PTRS and add
             # the object to the list 'old_objects_pointing_to_young'.
             # We know that 'addr_struct' cannot be in the nursery,
@@ -1593,6 +1561,13 @@
 
 
     def collect_oldrefs_to_nursery(self):
+        if self.gc_state == STATE_MARKING:
+            self._collect_oldrefs_to_nursery(True)
+        else:
+            self._collect_oldrefs_to_nursery(False)
+
+    @specialize.arg(1)
+    def _collect_oldrefs_to_nursery(self, state_is_marking):
         # Follow the old_objects_pointing_to_young list and move the
         # young objects they point to out of the nursery.
         oldlist = self.old_objects_pointing_to_young
@@ -1609,6 +1584,14 @@
             # have this flag set after a nursery collection.
             self.header(obj).tid |= GCFLAG_TRACK_YOUNG_PTRS
             #
+            # If the incremental major collection is currently at
+            # STATE_MARKING, then we must add to 'objects_to_trace' all
+            # black objects that go through 'old_objects_pointing_to_young'.
+            if state_is_marking and self.header(obj).tid & GCFLAG_VISITED != 0:
+                self.header(obj).tid &= ~GCFLAG_VISITED
+                self.header(obj).tid |= GCFLAG_GRAY
+                self.objects_to_trace.append(obj)
+            #
             # Trace the 'obj' to replace pointers to nursery with pointers
             # outside the nursery, possibly forcing nursery objects out
             # and adding them to 'old_objects_pointing_to_young' as well.
@@ -1650,11 +1633,6 @@
                 self._visit_young_rawmalloced_object(obj)
             return
         #
-
-        # Do this after check we are old to avoid cache misses like
-        # In the comment above.
-        self.header(obj).tid |= GCFLAG_OLD
-
         size_gc_header = self.gcheaderbuilder.size_gc_header
         if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
             #
@@ -1711,10 +1689,6 @@
         if self.has_gcptr(typeid):
             # we only have to do it if we have any gcptrs
             self.old_objects_pointing_to_young.append(newobj)
-        #
-        # If we are in STATE_MARKING, then the new object must be made gray.
-        if self.gc_state == STATE_MARKING:
-            self.write_to_visited_object_backward(newobj)
 
     _trace_drag_out._always_inline_ = True
 
@@ -1729,7 +1703,7 @@
         hdr = self.header(obj)
         if hdr.tid & GCFLAG_VISITED:
             return
-        hdr.tid |= (GCFLAG_VISITED|GCFLAG_OLD)
+        hdr.tid |= GCFLAG_VISITED
         #
         # we just made 'obj' old, so we need to add it to the correct lists
         added_somewhere = False
@@ -1782,9 +1756,7 @@
     def _free_young_rawmalloced_obj(self, obj, ignored1, ignored2):
         # If 'obj' has GCFLAG_VISITED, it was seen by _trace_drag_out
         # and survives.  Otherwise, it dies.
-        if not self.free_rawmalloced_object_if_unvisited(obj):
-            if self.gc_state == STATE_MARKING:
-                self.write_to_visited_object_backward(obj)
+        self.free_rawmalloced_object_if_unvisited(obj)
 
     def remove_young_arrays_from_old_objects_pointing_to_young(self):
         old = self.old_objects_pointing_to_young
@@ -1799,7 +1771,7 @@
             old.append(new.pop())
         new.delete()
 
-    def gc_step_until(self,state):
+    def gc_step_until(self, state):
         while self.gc_state != state:
             self.minor_collection()
             self.major_collection_step()
@@ -1814,7 +1786,7 @@
 
     # Note - minor collections seem fast enough so that one
     # is done before every major collection step
-    def major_collection_step(self,reserving_size=0):
+    def major_collection_step(self, reserving_size=0):
         debug_start("gc-collect-step")
         debug_print("stating gc state: ", GC_STATES[self.gc_state])
         # Debugging checks
@@ -1830,7 +1802,7 @@
             # XXX do it in one step
             self.collect_roots()
             #set all found roots to gray before entering marking state
-            self.objects_to_trace.foreach(self._set_gcflag_gray,None)
+            self.objects_to_trace.foreach(self._set_gcflag_gray, None)
             self.gc_state = STATE_MARKING
             #END SCANNING
         elif self.gc_state == STATE_MARKING:
@@ -1950,10 +1922,8 @@
 
     def free_rawmalloced_object_if_unvisited(self, obj):
         if self.header(obj).tid & GCFLAG_VISITED:
-            self.header(obj).tid |= GCFLAG_OLD
             self.header(obj).tid &= ~(GCFLAG_VISITED|GCFLAG_GRAY)   # survives
             self.old_rawmalloced_objects.append(obj)
-            return False
         else:
             size_gc_header = self.gcheaderbuilder.size_gc_header
             totalsize = size_gc_header + self.get_size(obj)
@@ -1976,7 +1946,6 @@
             #
             llarena.arena_free(arena)
             self.rawmalloced_total_size -= r_uint(allocsize)
-            return True
 
     def start_free_rawmalloc_objects(self):
         self.raw_malloc_might_sweep = self.old_rawmalloced_objects
@@ -2071,7 +2040,7 @@
             return
         #
         # It's the first time.  We set the flag.
-        hdr.tid |= GCFLAG_VISITED
+        hdr.tid |= GCFLAG_VISITED | GCFLAG_TRACK_YOUNG_PTRS
 
 
         if not self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)):

From noreply at buildbot.pypy.org  Fri Oct 11 16:51:33 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 16:51:33 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: more in-progress
Message-ID: <20131011145133.C54031C0203@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67318:9eacdacab7e5
Date: 2013-10-11 16:50 +0200
http://bitbucket.org/pypy/pypy/changeset/9eacdacab7e5/

Log:	more in-progress

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -124,12 +124,7 @@
 # note that GCFLAG_CARDS_SET is the most significant bit of a byte:
 # this is required for the JIT (x86)
 
-# This flag is used by the tri color algorithm. An object which
-# has the gray bit set has been marked reachable, but not yet walked
-# by the incremental collection
-GCFLAG_GRAY         = first_gcflag << 8
-
-_GCFLAG_FIRST_UNUSED = first_gcflag << 9    # the first unused bit
+_GCFLAG_FIRST_UNUSED = first_gcflag << 8    # the first unused bit
 
 
 # States for the incremental GC
@@ -138,22 +133,18 @@
 # This state must complete in a single step
 STATE_SCANNING = 0
 
-# The marking phase. We walk the list of all grey objects and mark
-# all of the things they point to grey. This step lasts until there are no
-# gray objects
-STATE_MARKING  = 1
+# The marking phase. We walk the list 'objects_to_trace' of all gray objects
+# and mark all of the things they point to gray. This step lasts until there
+# are no more gray objects.
+STATE_MARKING = 1
 
-# here we kill all the unvisited rawmalloc objects
-STATE_SWEEPING_RAWMALLOC = 2
-
-# here we kill all the unvisited arena objects
-STATE_SWEEPING_ARENA = 3
+# here we kill all the unvisited objects
+STATE_SWEEPING = 2
 
 # here we call all the finalizers
-STATE_FINALIZING = 4
+STATE_FINALIZING = 3
 
-GC_STATES = ['SCANNING', 'MARKING', 'SWEEPING_RAWMALLOC', 'SWEEPING_ARENA',
-             'FINALIZING']
+GC_STATES = ['SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING']
 
 
 FORWARDSTUB = lltype.GcStruct('forwarding_stub',
@@ -339,6 +330,7 @@
         # Two lists of all raw_malloced objects (the objects too large)
         self.young_rawmalloced_objects = self.null_address_dict()
         self.old_rawmalloced_objects = self.AddressStack()
+        self.raw_malloc_might_sweep = self.AddressStack()
         self.rawmalloced_total_size = r_uint(0)
 
         self.gc_state = STATE_SCANNING
@@ -752,11 +744,10 @@
         #
         # If somebody calls this function a lot, we must eventually
         # force a full collection.
-        # XXX REDO
-##        if (float(self.get_total_memory_used()) + raw_malloc_usage(totalsize) >
-##                self.next_major_collection_threshold):
-##            self.minor_collection()
-##            self.major_collection(raw_malloc_usage(totalsize))
+        if (float(self.get_total_memory_used()) + raw_malloc_usage(totalsize) >
+                self.next_major_collection_threshold):
+            self.gc_step_until(STATE_SWEEPING)
+            self.gc_step_until(STATE_FINALIZING, raw_malloc_usage(totalsize))
         #
         # Check if the object would fit in the ArenaCollection.
         if raw_malloc_usage(totalsize) <= self.small_request_threshold:
@@ -1049,9 +1040,7 @@
                 MovingGCBase.debug_check_consistency(self)
                 self._debug_objects_to_trace_dict.delete()
                 already_checked = True
-            elif self.gc_state == STATE_SWEEPING_RAWMALLOC:
-                pass
-            elif self.gc_state == STATE_SWEEPING_ARENA:
+            elif self.gc_state == STATE_SWEEPING:
                 pass
             elif self.gc_state == STATE_FINALIZING:
                 pass
@@ -1072,10 +1061,8 @@
             self._debug_check_object_scanning(obj)
         elif self.gc_state == STATE_MARKING:
             self._debug_check_object_marking(obj)
-        elif self.gc_state == STATE_SWEEPING_RAWMALLOC:
-            self._debug_check_object_sweeping_rawmalloc(obj)
-        elif self.gc_state == STATE_SWEEPING_ARENA:
-            self._debug_check_object_sweeping_arena(obj)
+        elif self.gc_state == STATE_SWEEPING:
+            self._debug_check_object_sweeping(obj)
         elif self.gc_state == STATE_FINALIZING:
             self._debug_check_object_finalizing(obj)
         else:
@@ -1106,10 +1093,7 @@
         ll_assert(self.header(obj).tid & (GCFLAG_GRAY | GCFLAG_VISITED) != 0,
                   "visited object points to unprocessed (white) object." )
 
-    def _debug_check_object_sweeping_rawmalloc(self, obj):
-        pass
-
-    def _debug_check_object_sweeping_arena(self, obj):
+    def _debug_check_object_sweeping(self, obj):
         pass
 
     def _debug_check_object_finalizing(self,obj):
@@ -1203,21 +1187,6 @@
         if self.header(addr_array).tid & GCFLAG_TRACK_YOUNG_PTRS:
             self.remember_young_pointer(addr_array)
 
-    def write_to_visited_object_backward(self, addr_struct):
-        """Call during the marking phase only, when writing into an object
-        that is 'black' in terms of the classical tri-color GC, i.e. that
-        has the GCFLAG_VISITED.  This implements a 'backward' write barrier,
-        i.e. it turns the object back from 'black' to 'gray'.
-        """
-        ll_assert(self.gc_state == STATE_MARKING,"expected MARKING state")
-        # writing a white object into black, make black gray and
-        # readd to objects_to_trace
-        # this is useful for arrays because it stops the writebarrier
-        # from being re-triggered on successive writes
-        self.header(addr_struct).tid &= ~GCFLAG_VISITED
-        self.header(addr_struct).tid |= GCFLAG_GRAY
-        self.objects_to_trace.append(addr_struct)
-
     def _init_writebarrier_logic(self):
         DEBUG = self.DEBUG
         # The purpose of attaching remember_young_pointer to the instance
@@ -1689,7 +1658,6 @@
         if self.has_gcptr(typeid):
             # we only have to do it if we have any gcptrs
             self.old_objects_pointing_to_young.append(newobj)
-
     _trace_drag_out._always_inline_ = True
 
     def _visit_young_rawmalloced_object(self, obj):
@@ -1771,10 +1739,10 @@
             old.append(new.pop())
         new.delete()
 
-    def gc_step_until(self, state):
+    def gc_step_until(self, state, reserving_size=0):
         while self.gc_state != state:
             self.minor_collection()
-            self.major_collection_step()
+            self.major_collection_step(reserving_size)
 
     debug_gc_step_until = gc_step_until   # xxx
 
@@ -1829,29 +1797,27 @@
                 if self.old_objects_with_light_finalizers.non_empty():
                     self.deal_with_old_objects_with_finalizers()
                 #objects_to_trace processed fully, can move on to sweeping
-                self.gc_state = STATE_SWEEPING_RAWMALLOC
+                self.gc_state = STATE_SWEEPING
                 #prepare for the next state
                 self.ac.mass_free_prepare()
                 self.start_free_rawmalloc_objects()
             #END MARKING
-        elif self.gc_state == STATE_SWEEPING_RAWMALLOC:
+        elif self.gc_state == STATE_SWEEPING:
             #
             # Walk all rawmalloced objects and free the ones that don't
-            # have the GCFLAG_VISITED flag.
-            # XXX heuristic here to decide nobjects.
-            nobjects = self.nursery_size // self.ac.page_size  # XXX
-            if self.free_unvisited_rawmalloc_objects_step(nobjects):
-                #malloc objects freed
-                self.gc_state = STATE_SWEEPING_ARENA
-
-        elif self.gc_state == STATE_SWEEPING_ARENA:
+            # have the GCFLAG_VISITED flag.  Visit at most 'limit' objects.
+            limit = self.nursery_size // self.ac.page_size
+            remaining = self.free_unvisited_rawmalloc_objects_step(limit)
             #
-            # Ask the ArenaCollection to visit all objects.  Free the ones
-            # that have not been visited above, and reset GCFLAG_VISITED on
-            # the others.
-            max_pages = 3 * (self.nursery_size // self.ac.page_size)  # XXX
-            if self.ac.mass_free_incremental(self._free_if_unvisited,
-                                             max_pages):
+            # Ask the ArenaCollection to visit a fraction of the objects.
+            # Free the ones that have not been visited above, and reset
+            # GCFLAG_VISITED on the others.  Visit at most '3 * limit'
+            # pages minus the number of objects already visited above.
+            done = self.ac.mass_free_incremental(self._free_if_unvisited,
+                                                 2 * limit + remaining)
+            # XXX tweak the limits above
+            #
+            if remaining == 0 and done:
                 self.num_major_collects += 1
                 #
                 # We also need to reset the GCFLAG_VISITED on prebuilt GC objects.
@@ -1948,21 +1914,17 @@
             self.rawmalloced_total_size -= r_uint(allocsize)
 
     def start_free_rawmalloc_objects(self):
-        self.raw_malloc_might_sweep = self.old_rawmalloced_objects
-        self.old_rawmalloced_objects = self.AddressStack()
+        (self.raw_malloc_might_sweep, self.old_rawmalloced_objects) = (
+            self.old_rawmalloced_objects, self.raw_malloc_might_sweep)
 
     # Returns true when finished processing objects
-    def free_unvisited_rawmalloc_objects_step(self,nobjects=1):
-
-        while nobjects > 0 and self.raw_malloc_might_sweep.non_empty():
+    def free_unvisited_rawmalloc_objects_step(self, nobjects):
+        while self.raw_malloc_might_sweep.non_empty() and nobjects > 0:
             self.free_rawmalloced_object_if_unvisited(
                                              self.raw_malloc_might_sweep.pop())
             nobjects -= 1
 
-        if not self.raw_malloc_might_sweep.non_empty():
-            self.raw_malloc_might_sweep.delete()
-            return True
-        return False
+        return nobjects
 
 
     def collect_roots(self):
@@ -1997,13 +1959,7 @@
         self.objects_to_trace.append(obj)
 
     def _collect_ref_rec(self, root, ignored):
-        obj = root.address[0]
-        # XXX minimark.py doesn't read anything from 'obj' here.
-        # Can this lead to seriously more cache pressure?
-        if self.header(obj).tid & (GCFLAG_VISITED|GCFLAG_GRAY) != 0:
-            return
-        self.header(obj).tid |= GCFLAG_GRAY
-        self.objects_to_trace.append(obj)
+        self.objects_to_trace.append(root.address[0])
 
     def visit_all_objects(self):
         pending = self.objects_to_trace
diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -357,7 +357,10 @@
             #
             size_class -= 1
         #
-        self._rehash_arenas_lists()
+        if size_class >= 0:
+            self._rehash_arenas_lists()
+            self.size_class_with_old_pages = -1
+        #
         return True
 
 

From noreply at buildbot.pypy.org  Fri Oct 11 17:13:04 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri, 11 Oct 2013 17:13:04 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: a failing test and the
 corresponding fix. I hate RPython,
 this corresponds to a segfault after translation :-/
Message-ID: <20131011151304.1C2101C021C@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67319:0cffbb8df165
Date: 2013-10-11 17:12 +0200
http://bitbucket.org/pypy/pypy/changeset/0cffbb8df165/

Log:	a failing test and the corresponding fix. I hate RPython, this
	corresponds to a segfault after translation :-/

diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -42,6 +42,12 @@
     def is_char_or_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveCharOrUniChar)
 
+    def aslist_int(self, cdata):
+        return None
+
+    def aslist_float(self, cdata):
+        return None
+
     def cast(self, w_ob):
         # cast to a pointer, to a funcptr, or to an array.
         # Note that casting to an array is an extension to the C language,
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -75,6 +75,16 @@
         if not self.runappdirect:
             assert self.get_count() == 1
 
+    def test_TypeError_if_no_length(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, 3)
+        buf = _cffi_backend.newp(LONG_ARRAY)
+        pbuf = _cffi_backend.cast(P_LONG, buf)
+        raises(TypeError, "list(pbuf)")
+
+
     def test_list_float(self):
         import _cffi_backend
         DOUBLE = _cffi_backend.new_primitive_type('double')

From noreply at buildbot.pypy.org  Fri Oct 11 17:16:09 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 17:16:09 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: more in-progress
Message-ID: <20131011151609.5D2911C021C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67320:656c62aad308
Date: 2013-10-11 17:15 +0200
http://bitbucket.org/pypy/pypy/changeset/656c62aad308/

Log:	more in-progress

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1020,42 +1020,29 @@
              (self.card_page_shift + 3)))
 
     def debug_check_consistency(self):
+        if self.DEBUG:
+            ll_assert(not self.young_rawmalloced_objects,
+                      "young raw-malloced objects in a major collection")
+            ll_assert(not self.young_objects_with_weakrefs.non_empty(),
+                      "young objects with weakrefs in a major collection")
 
-        if self.DEBUG:
+            if self.raw_malloc_might_sweep.non_empty():
+                ll_assert(self.gc_state == STATE_SWEEPING,
+                      "raw_malloc_might_sweep must be empty outside SWEEPING")
 
-            # somewhat of a hack
-            # some states require custom prep and cleanup
-            # before calling the check_object functions
-            already_checked = False
-
-            if self.gc_state == STATE_SCANNING:
-                # We are just starting a scan. Same as a non incremental here.
-                ll_assert(not self.young_rawmalloced_objects,
-                          "young raw-malloced objects in a major collection")
-                ll_assert(not self.young_objects_with_weakrefs.non_empty(),
-                          "young objects with weakrefs in a major collection")
-            elif self.gc_state == STATE_MARKING:
+            if self.gc_state == STATE_MARKING:
                 self._debug_objects_to_trace_dict = \
                                             self.objects_to_trace.stack2dict()
                 MovingGCBase.debug_check_consistency(self)
                 self._debug_objects_to_trace_dict.delete()
-                already_checked = True
-            elif self.gc_state == STATE_SWEEPING:
-                pass
-            elif self.gc_state == STATE_FINALIZING:
-                pass
             else:
-                ll_assert(False,"uknown gc_state value")
-
-            if not already_checked:
                 MovingGCBase.debug_check_consistency(self)
 
-
     def debug_check_object(self, obj):
-
-        ll_assert((self.header(obj).tid & GCFLAG_GRAY != 0
-                    and self.header(obj).tid & GCFLAG_VISITED != 0) == False,
-                    "object gray and visited at the same time." )
+        # We are after a minor collection, and possibly after a major
+        # collection step.  No object should be in the nursery
+        ll_assert(not self.is_in_nursery(obj),
+                  "object in nursery after collection")
 
         if self.gc_state == STATE_SCANNING:
             self._debug_check_object_scanning(obj)
@@ -1066,60 +1053,39 @@
         elif self.gc_state == STATE_FINALIZING:
             self._debug_check_object_finalizing(obj)
         else:
-            ll_assert(False,"uknown gc_state value")
+            ll_assert(False, "unknown gc_state value")
 
     def _debug_check_object_marking(self, obj):
         if self.header(obj).tid & GCFLAG_VISITED != 0:
-            # Visited, should NEVER point to a white object.
-            self.trace(obj,self._debug_check_not_white,None)
+            # A black object.  Should NEVER point to a white object.
+            self.trace(obj, self._debug_check_not_white, None)
             # During marking, all visited (black) objects should always have
             # the GCFLAG_TRACK_YOUNG_PTRS flag set, for the write barrier to
-            # trigger
-            ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS != 0,
-                      "black object without GCFLAG_TRACK_YOUNG_PTRS")
-
-        if self.header(obj).tid & GCFLAG_GRAY != 0:
-            ll_assert(self._debug_objects_to_trace_dict.contains(obj),
-                        "gray object not in pending trace list.")
-        else:
-            #if not gray and not black
-            if self.header(obj).tid & GCFLAG_VISITED == 0:
-                if self.header(obj).tid & GCFLAG_NO_HEAP_PTRS == 0:
-                    ll_assert(not self._debug_objects_to_trace_dict.contains(obj),
-                        "white object in pending trace list.")
+            # trigger --- at least if they contain any gc ptr
+            typeid = self.get_type_id(obj)
+            if self.has_gcptr(typeid):
+                ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS != 0,
+                          "black object without GCFLAG_TRACK_YOUNG_PTRS")
 
     def _debug_check_not_white(self, root, ignored):
         obj = root.address[0]
-        ll_assert(self.header(obj).tid & (GCFLAG_GRAY | GCFLAG_VISITED) != 0,
-                  "visited object points to unprocessed (white) object." )
+        if self.header(obj).tid & GCFLAG_VISITED != 0:
+            pass    # black -> black
+        elif self._debug_objects_to_trace_dict.contains(obj):
+            pass    # black -> gray
+        else:
+            ll_assert(False, "black -> white pointer found")
 
     def _debug_check_object_sweeping(self, obj):
-        pass
+        # We see only reachable objects here.  They all start as VISITED
+        # but this flag is progressively removed in the sweeping phase.
 
-    def _debug_check_object_finalizing(self,obj):
-        pass
-
-    def _debug_check_object_scanning(self, obj):
-        # This check is called before scanning starts.
-        # scanning is done in a single step.
-
-        # after a minor or major collection, no object should be in the nursery
-        ll_assert(not self.is_in_nursery(obj),
-                  "object in nursery after collection")
-        # similarily, all objects should have this flag, except if they
+        # All objects should have this flag, except if they
         # don't have any GC pointer
         typeid = self.get_type_id(obj)
         if self.has_gcptr(typeid):
             ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS != 0,
                       "missing GCFLAG_TRACK_YOUNG_PTRS")
-        # the GCFLAG_VISITED should not be set between collections
-        ll_assert(self.header(obj).tid & GCFLAG_VISITED == 0,
-                  "unexpected GCFLAG_VISITED")
-
-        # the GCFLAG_GRAY should never be set at the start of a collection
-        ll_assert(self.header(obj).tid & GCFLAG_GRAY == 0,
-          "unexpected GCFLAG_GRAY")
-
         # the GCFLAG_FINALIZATION_ORDERING should not be set between coll.
         ll_assert(self.header(obj).tid & GCFLAG_FINALIZATION_ORDERING == 0,
                   "unexpected GCFLAG_FINALIZATION_ORDERING")
@@ -1149,6 +1115,22 @@
                           "the card marker bits are not cleared")
                 i -= 1
 
+    def _debug_check_object_finalizing(self, obj):
+        # Same invariants as STATE_SCANNING.
+        self._debug_check_object_scanning(obj)
+
+    def _debug_check_object_scanning(self, obj):
+        # This check is called before scanning starts.
+        # Scanning is done in a single step.
+        # the GCFLAG_VISITED should not be set between collections
+        ll_assert(self.header(obj).tid & GCFLAG_VISITED == 0,
+                  "unexpected GCFLAG_VISITED")
+
+        # All other invariants from the sweeping phase should still be
+        # satisfied.
+        self._debug_check_object_sweeping(obj)
+
+
     # ----------
     # Write barrier
 
@@ -1767,10 +1749,7 @@
         # to smaller increments using stacks for resuming
         if self.gc_state == STATE_SCANNING:
             self.objects_to_trace = self.AddressStack()
-            # XXX do it in one step
             self.collect_roots()
-            #set all found roots to gray before entering marking state
-            self.objects_to_trace.foreach(self._set_gcflag_gray, None)
             self.gc_state = STATE_MARKING
             #END SCANNING
         elif self.gc_state == STATE_MARKING:
@@ -1797,10 +1776,9 @@
                 if self.old_objects_with_light_finalizers.non_empty():
                     self.deal_with_old_objects_with_finalizers()
                 #objects_to_trace processed fully, can move on to sweeping
-                self.gc_state = STATE_SWEEPING
-                #prepare for the next state
                 self.ac.mass_free_prepare()
                 self.start_free_rawmalloc_objects()
+                self.gc_state = STATE_SWEEPING
             #END MARKING
         elif self.gc_state == STATE_SWEEPING:
             #
@@ -1883,9 +1861,6 @@
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~(GCFLAG_VISITED|GCFLAG_GRAY)
 
-    def _set_gcflag_gray(self, obj, ignored):
-        self.header(obj).tid |= GCFLAG_GRAY
-
     def free_rawmalloced_object_if_unvisited(self, obj):
         if self.header(obj).tid & GCFLAG_VISITED:
             self.header(obj).tid &= ~(GCFLAG_VISITED|GCFLAG_GRAY)   # survives
@@ -1962,19 +1937,13 @@
         self.objects_to_trace.append(root.address[0])
 
     def visit_all_objects(self):
-        pending = self.objects_to_trace
-        while pending.non_empty():
-            obj = pending.pop()
-            self.visit(obj)
+        self.visit_all_objects_step(sys.maxint)
 
     def visit_all_objects_step(self, nobjects):
         # Objects can be added to pending by visit
         pending = self.objects_to_trace
         while nobjects > 0 and pending.non_empty():
             obj = pending.pop()
-            ll_assert(self.header(obj).tid &
-                        (GCFLAG_GRAY|GCFLAG_VISITED|GCFLAG_NO_HEAP_PTRS) != 0,
-                        "non gray or black object being traced")
             self.visit(obj)
             nobjects -= 1
 
@@ -1990,15 +1959,13 @@
         # and the GCFLAG_VISITED will be reset at the end of the
         # collection.
         hdr = self.header(obj)
-        # visited objects are no longer grey
-        hdr.tid &= ~GCFLAG_GRAY
         if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
             return
         #
-        # It's the first time.  We set the flag.
+        # It's the first time.  We set the flag VISITED.  The trick is
+        # to also set TRACK_YOUNG_PTRS here, for the write barrier.
         hdr.tid |= GCFLAG_VISITED | GCFLAG_TRACK_YOUNG_PTRS
 
-
         if not self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)):
             return
         #

From noreply at buildbot.pypy.org  Fri Oct 11 18:28:09 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 11 Oct 2013 18:28:09 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: forgotten in last commit
Message-ID: <20131011162809.0DA9A1D22C2@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67322:37ac9e5dd2d7
Date: 2013-10-11 13:31 +0200
http://bitbucket.org/pypy/pypy/changeset/37ac9e5dd2d7/

Log:	forgotten in last commit

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -705,7 +705,7 @@
                           frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE)
         if logger:
             logger.log_bridge(inputargs, operations, "rewritten",
-                              ops_offset=ops_offset)
+                              ops_offset=ops_offset, descr=faildescr)
         self.fixup_target_tokens(rawstart)
         self.update_frame_depth(frame_depth)
         self.teardown()

From noreply at buildbot.pypy.org  Fri Oct 11 18:28:07 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 11 Oct 2013 18:28:07 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: test and fix extract_category for
 rewritten and noopt traces
Message-ID: <20131011162807.AAD011C3654@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67321:3710d2d308ba
Date: 2013-10-11 13:30 +0200
http://bitbucket.org/pypy/pypy/changeset/3710d2d308ba/

Log:	test and fix extract_category for rewritten and noopt traces

diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -374,19 +374,16 @@
             i += 1
     return res
 
-def purge_thread_numbers(entry):
-    result = []
-    for line in entry.split('\n'):
-        line = line[line.find('#')+2:]
-        result.append(line)
-    return '\n'.join(result)
+r_skip_thread = re.compile(r'^(\d+#)?')
+def skip_thread_numbers(entry):
+    return r_skip_thread.sub('', entry).strip()
     
 def import_log(logname, ParserCls=SimpleParser):
     log = parse_log_file(logname)
     hex_re = '0x(-?[\da-f]+)'
     addrs = {}
     for entry in extract_category(log, 'jit-backend-addr'):
-        entry = purge_thread_numbers(entry)
+        entry = skip_thread_numbers(entry)
         m = re.search('bootstrap ' + hex_re, entry)
         if not m:
             # a bridge
@@ -402,7 +399,7 @@
     from rpython.jit.backend.tool.viewcode import World
     world = World()
     for entry in extract_category(log, 'jit-backend-dump'):
-        entry = purge_thread_numbers(entry)
+        entry = skip_thread_numbers(entry)
         world.parse(entry.splitlines(True))
     dumps = {}
     symbols = world.symbols
@@ -414,9 +411,9 @@
     loops = []
     cat = extract_category(log, 'jit-log-opt')
     if not cat:
-        extract_category(log, 'jit-log-rewritten')
+        cat = extract_category(log, 'jit-log-rewritten')
     if not cat:
-        extract_category(log, 'jit-log-noopt')        
+        cat = extract_category(log, 'jit-log-noopt')        
     for entry in cat:
         parser = ParserCls(entry, None, {}, 'lltype', None,
                            nonstrict=True)
@@ -473,7 +470,7 @@
         mapping[loop.descr] = loop
     for line in lines:
         if line:
-            line = purge_thread_numbers(line)
+            line = skip_thread_numbers(line)
             num, count = line.split(':', 2)
             mapping[num].count = int(count)
 
diff --git a/pypy/tool/jitlogparser/test/logtest.log b/pypy/tool/jitlogparser/test/logtest.log
--- a/pypy/tool/jitlogparser/test/logtest.log
+++ b/pypy/tool/jitlogparser/test/logtest.log
@@ -1,38 +1,38 @@
-[11f210b47027] {jit-backend
-[11f210b900f7] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f3b0b2e63d5 +0  554889E5534154415541564157488DA500000000488B042590C5540148C7042590C554010000000048898570FFFFFF488B042598C5540148C7042598C554010000000048898568FFFFFF488B0425A0C5540148C70425A0C554010000000048898560FFFFFF488B0425A8C5540148C70425A8C554010000000048898558FFFFFF4C8B3C2550525B0149BB3050920D3B7F00004D8B334983C60149BB3050920D3B7F00004D89334981FF102700000F8D000000004983C7014C8B342580F76A024983EE014C89342580F76A024983FE000F8C00000000E9AEFFFFFF488B042588F76A024829E0483B042580EC3C01760D49BB05632E0B3B7F000041FFD3554889E5534154415541564157488DA550FFFFFF4889BD70FFFFFF4889B568FFFFFF48899560FFFFFF48898D58FFFFFF4D89C7E954FFFFFF49BB00602E0B3B7F000041FFD34440484C3D030300000049BB00602E0B3B7F000041FFD34440484C3D070304000000
-[11f210b949b3] jit-backend-dump}
-[11f210b949b4] {jit-backend-addr
-Loop 0 (<code object f, file 'x.py', line 2> #9 LOAD_FAST) has address 0x7f3b0b2e645d to 0x7f3b0b2e64af (bootstrap 0x7f3b0b2e63d5)
-[11f210bab188] jit-backend-addr}
-[11f210bab189] jit-backend}
-[11f210bacbb7] {jit-log-opt-loop
-# Loop 0 : loop with 19 ops
-[p0, p1, p2, p3, i4]
-debug_merge_point(0, '<code object f. file 'x.py'. line 2> #9 LOAD_FAST')
-debug_merge_point(0, '<code object f. file 'x.py'. line 2> #12 LOAD_CONST')
-debug_merge_point(0, '<code object f. file 'x.py'. line 2> #15 COMPARE_OP')
-+166: i6 = int_lt(i4, 10000)
-guard_true(i6, descr=<Guard0x3>) [p1, p0, p2, p3, i4]
-debug_merge_point(0, '<code object f. file 'x.py'. line 2> #18 POP_JUMP_IF_FALSE')
-debug_merge_point(0, '<code object f. file 'x.py'. line 2> #21 LOAD_FAST')
-debug_merge_point(0, '<code object f. file 'x.py'. line 2> #24 LOAD_CONST')
-debug_merge_point(0, '<code object f. file 'x.py'. line 2> #27 INPLACE_ADD')
-+179: i8 = int_add(i4, 1)
-debug_merge_point(0, '<code object f. file 'x.py'. line 2> #28 STORE_FAST')
-debug_merge_point(0, '<code object f. file 'x.py'. line 2> #31 JUMP_ABSOLUTE')
-+183: i10 = getfield_raw(40564608, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
-+191: i12 = int_sub(i10, 1)
-+195: setfield_raw(40564608, i12, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
-+203: i14 = int_lt(i12, 0)
-guard_false(i14, descr=<Guard0x4>) [p1, p0, p2, p3, i8, None]
-debug_merge_point(0, '<code object f. file 'x.py'. line 2> #9 LOAD_FAST')
-+213: jump(p0, p1, p2, p3, i8, descr=<Loop0>)
-+218: --end of the loop--
-[11f210c17981] jit-log-opt-loop}
-[11f210fb1d21] {jit-backend-counts
-0:8965
-1:2
-[11f210fb381b] jit-backend-counts}
+0# [11f210b47027] {jit-backend
+0# [11f210b900f7] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f3b0b2e63d5 +0  554889E5534154415541564157488DA500000000488B042590C5540148C7042590C554010000000048898570FFFFFF488B042598C5540148C7042598C554010000000048898568FFFFFF488B0425A0C5540148C70425A0C554010000000048898560FFFFFF488B0425A8C5540148C70425A8C554010000000048898558FFFFFF4C8B3C2550525B0149BB3050920D3B7F00004D8B334983C60149BB3050920D3B7F00004D89334981FF102700000F8D000000004983C7014C8B342580F76A024983EE014C89342580F76A024983FE000F8C00000000E9AEFFFFFF488B042588F76A024829E0483B042580EC3C01760D49BB05632E0B3B7F000041FFD3554889E5534154415541564157488DA550FFFFFF4889BD70FFFFFF4889B568FFFFFF48899560FFFFFF48898D58FFFFFF4D89C7E954FFFFFF49BB00602E0B3B7F000041FFD34440484C3D030300000049BB00602E0B3B7F000041FFD34440484C3D070304000000
+0# [11f210b949b3] jit-backend-dump}
+0# [11f210b949b4] {jit-backend-addr
+0# Loop 0 (<code object f, file 'x.py', line 2> #9 LOAD_FAST) has address 0x7f3b0b2e645d to 0x7f3b0b2e64af (bootstrap 0x7f3b0b2e63d5)
+0# [11f210bab188] jit-backend-addr}
+0# [11f210bab189] jit-backend}
+0# [11f210bacbb7] {jit-log-opt-loop
+0# # Loop 0 : loop with 19 ops
+0# [p0, p1, p2, p3, i4]
+0# debug_merge_point(0, '<code object f. file 'x.py'. line 2> #9 LOAD_FAST')
+0# debug_merge_point(0, '<code object f. file 'x.py'. line 2> #12 LOAD_CONST')
+0# debug_merge_point(0, '<code object f. file 'x.py'. line 2> #15 COMPARE_OP')
+0# +166: i6 = int_lt(i4, 10000)
+0# guard_true(i6, descr=<Guard0x3>) [p1, p0, p2, p3, i4]
+0# debug_merge_point(0, '<code object f. file 'x.py'. line 2> #18 POP_JUMP_IF_FALSE')
+0# debug_merge_point(0, '<code object f. file 'x.py'. line 2> #21 LOAD_FAST')
+0# debug_merge_point(0, '<code object f. file 'x.py'. line 2> #24 LOAD_CONST')
+0# debug_merge_point(0, '<code object f. file 'x.py'. line 2> #27 INPLACE_ADD')
+0# +179: i8 = int_add(i4, 1)
+0# debug_merge_point(0, '<code object f. file 'x.py'. line 2> #28 STORE_FAST')
+0# debug_merge_point(0, '<code object f. file 'x.py'. line 2> #31 JUMP_ABSOLUTE')
+0# +183: i10 = getfield_raw(40564608, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
+0# +191: i12 = int_sub(i10, 1)
+0# +195: setfield_raw(40564608, i12, descr=<SignedFieldDescr pypysig_long_struct.c_value 0>)
+0# +203: i14 = int_lt(i12, 0)
+0# guard_false(i14, descr=<Guard0x4>) [p1, p0, p2, p3, i8, None]
+0# debug_merge_point(0, '<code object f. file 'x.py'. line 2> #9 LOAD_FAST')
+0# +213: jump(p0, p1, p2, p3, i8, descr=<Loop0>)
+0# +218: --end of the loop--
+0# [11f210c17981] jit-log-opt-loop}
+0# [11f210fb1d21] {jit-backend-counts
+0# 0:8965
+0# 1:2
+0# [11f210fb381b] jit-backend-counts}
diff --git a/pypy/tool/jitlogparser/test/logtest2.log b/pypy/tool/jitlogparser/test/logtest2.log
--- a/pypy/tool/jitlogparser/test/logtest2.log
+++ b/pypy/tool/jitlogparser/test/logtest2.log
@@ -1,356 +1,356 @@
-[1cffd8feb691] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31000 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
-[1cffd8ffaba6] jit-backend-dump}
-[1cffd90012ee] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31085 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000488B1C2508E6150348C7042500E615030000000048C7042508E615030000000048895D38584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
-[1cffd9003b76] jit-backend-dump}
-[1cffd900719f] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b3112e +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000488B7C240841BBB064120141FFD3488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000C20800
-[1cffd9008c81] jit-backend-dump}
-[1cffd900b384] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31191 +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000488B7C240841BBF065120141FFD3488B442408F6400480488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000C20800
-[1cffd900cf18] jit-backend-dump}
-[1cffd9010345] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b311fd +0  4883EC384889442408F20F114424184889EF48895C24284C89642430488B1C2508E615034C8B242500E6150348C7042500E615030000000048C7042508E615030000000041BBB064120141FFD3F20F10442418488B44240848891C2508E615034C89242500E61503488B5C24284C8B642430488D642438C3
-[1cffd9011f0b] jit-backend-dump}
-[1cffd9015bd8] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31275 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B4C240848894D20488B7424104889EF4883EC0848C7452810000000488B0C2508E6150348894D38488B1C2500E6150348C7042500E615030000000048C7042508E615030000000041BB60DBE80041FFD34889C5488B4D3848C745380000000048890C2508E6150348891C2500E615034883C40848C745280000000048C7452000000000488B4D58488B4560488B5568488B5D70488B7578488BBD800000004C8B85880000004C8B8D900000004C8B95980000004C8BA5A00000004C8BADA80000004C8BB5B00000004C8BBDB8000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C3
-[1cffd901a191] jit-backend-dump}
-[1cffd901b3a6] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31491 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
-[1cffd901dc46] jit-backend-dump}
-[1cffd901ef79] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31595 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B1C2508E6150348C7042500E615030000000048C7042508E615030000000048895D38584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
-[1cffd902ce01] jit-backend-dump}
-[1cffd902e819] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b316bd +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B7C240841BBB064120141FFD3488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C20800
-[1cffd9031b79] jit-backend-dump}
-[1cffd90331b0] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b3181e +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B7C240841BBF065120141FFD3488B442408F6400480488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C20800
-[1cffd903629a] jit-backend-dump}
-[1cffd903736b] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31988 +0  488B042508E6150348C7042500E615030000000048C7042508E61503000000004889453848C7451000C2B5014889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
-[1cffd9038a70] jit-backend-dump}
-[1cffd903e2cd] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b319e3 +0  48894D584889556848895D70488975784C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B4C240848894D204829C74883EC0848C745281000000041BBB0A4E80041FFD34883C4084885C00F84F4000000F645040174154883EC0849BBFD11B314557F000041FFD34883C40848C7452800000000488B4D58488B5568488B5D70488B75784C8B85880000004C8B8D900000004C8B95980000004C8BA5A00000004C8BADA80000004C8BB5B00000004C8BBDB8000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000488B3C25484CB60148C7452000000000C34883C40849BB8819B314557F000041FFE3
-[1cffd904265b] jit-backend-dump}
-[1cffd90448f2] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31bd6 +0  4889E74883EC0841BBD00F210141FFD34883C408488B042500E615034885C07501C34883C40849BB8819B314557F000041FFE3
-[1cffd9045d15] jit-backend-dump}
-[1cffd904647a] {jit-backend-counts
-[1cffd9046851] jit-backend-counts}
-[1cffd9636773] {jit-backend
-[1cffd9afbdde] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31ce0 +0  4881EC9800000048896C24604889FD48895C24684C896424704C896C24784C89B424800000004C89BC2488000000488B0425B07916034829E0483B0425109F0103760D49BBD61BB314557F000041FFD349BBF0509317557F00004D8B3B4D8D770149BBF0509317557F00004D89334C8BB5380100004D8BBE800000004D8B6E504D8B66704D0FB6968E0000004D8B4E604D8B4678498B7E58498B7668488B5E10488B5618488B4620488B4E284C89BD480100004C89AD500100004C898D580100004889BD60010000488995680100004889857001000048898D7801000049BB08519317557F0000498B0B488D410149BB08519317557F00004989034983F8020F85000000004883FB017206813B180C00000F85000000004983FA000F850000000049BB20803215557F00004D39DC0F85000000004C8B63084983FC0A0F8D00000000498D5C24014C8B2425807816034983FC000F8C0000000049BB20519317557F00004D8B234D8D54240149BB20519317557F00004D89134883FB0A0F8D000000004C8D5301488B1C25807816034883FB000F8C000000004C89D3E9B9FFFFFF49BB20E03215557F0000415349BB401CB314557F0000415349BB0010B314557F000041FFE349BB38C63815557F0000415349BB501CB314557F0000415349BB0010B314557F000041FFE349BBC0C53815557F0000415349BB601CB314557F0000415349BB0010B314557F000041FFE349BB48C53815557F0000415349BB701CB314557F0000415349BB0010B314557F000041FFE349BBD0C43815557F0000415349BB801CB314557F0000415349BB0010B314557F000041FFE349BB58C43815557F0000415349BB901CB314557F0000415349BB0010B314557F000041FFE349BBE0C33815557F0000415349BBA01CB314557F0000415349BB0010B314557F000041FFE349BB68C33815557F0000415349BBB01CB314557F0000415349BB0010B314557F000041FFE349BBF0C23815557F0000415349BBC01CB314557F0000415349BB0010B314557F000041FFE349BB78C23815557F0000415349BBD01CB314557F0000415349BB0010B314557F000041FFE3
-[1cffd9b146d6] jit-backend-dump}
-[1cffd9b14ff3] {jit-backend-addr
-Loop 0 (<code object g, file 'x.py', line 8> #9 LOAD_FAST) has address 0x7f5514b31d30 to 0x7f5514b31e80 (bootstrap 0x7f5514b31ce0)
-[1cffd9b16753] jit-backend-addr}
-[1cffd9b17245] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31de1 +0  9B000000
-[1cffd9b18103] jit-backend-dump}
-[1cffd9b18762] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31df3 +0  AE000000
-[1cffd9b191ae] jit-backend-dump}
-[1cffd9b1960b] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31dfd +0  C9000000
-[1cffd9b19f1f] jit-backend-dump}
-[1cffd9b1a32f] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31e10 +0  DB000000
-[1cffd9b1ac8d] jit-backend-dump}
-[1cffd9b1b091] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31e1e +0  F2000000
-[1cffd9b1ba54] jit-backend-dump}
-[1cffd9b1bfec] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31e35 +0  25010000
-[1cffd9b1c8d3] jit-backend-dump}
-[1cffd9b1ccfb] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31e5e +0  21010000
-[1cffd9b1d5d6] jit-backend-dump}
-[1cffd9b1da25] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b31e74 +0  55010000
-[1cffd9b1e3b8] jit-backend-dump}
-[1cffd9b1f0b2] jit-backend}
-[1cffd9b20d9f] {jit-log-opt-loop
-# Loop 0 (<code object g, file 'x.py', line 8> #9 LOAD_FAST) : loop with 59 ops
-[p0, p1]
-+110: p2 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_w_f_trace 128>)
-+124: p3 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_last_exception 80>)
-+128: p4 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_pycode 112>)
-+132: i5 = getfield_gc(p0, descr=<FieldU pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 142>)
-+140: p6 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_lastblock 96>)
-+144: i7 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_valuestackdepth 120>)
-+148: i8 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_last_instr 88>)
-+152: p9 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_locals_stack_w 104>)
-+156: p11 = getarrayitem_gc(p9, 0, descr=<ArrayP 8>)
-+160: p13 = getarrayitem_gc(p9, 1, descr=<ArrayP 8>)
-+164: p15 = getarrayitem_gc(p9, 2, descr=<ArrayP 8>)
-+168: p17 = getarrayitem_gc(p9, 3, descr=<ArrayP 8>)
-+172: p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_cells 40>)
-+172: label(p0, p1, p2, p3, p4, i5, p6, i7, i8, p11, p13, p15, p17, descr=TargetToken(140003404595232))
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
-+251: guard_value(i7, 2, descr=<Guard0x7f551532e020>) [i7, p1, p0, p2, p3, p4, i5, p6, i8, p11, p13, p15, p17]
-+261: guard_nonnull_class(p11, ConstClass(W_IntObject), descr=<Guard0x7f551538c638>) [p1, p0, p11, p2, p3, p4, i5, p6, p13, p15, p17]
-+279: guard_value(i5, 0, descr=<Guard0x7f551538c5c0>) [i5, p1, p0, p2, p3, p4, p6, p11, p13, p17]
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #12 LOAD_CONST')
-+289: guard_value(p4, ConstPtr(ptr22), descr=<Guard0x7f551538c548>) [p1, p0, p4, p2, p3, p6, p11, p13, p17]
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #15 COMPARE_OP')
-+308: i23 = getfield_gc_pure(p11, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval 8>)
-+312: i25 = int_lt(i23, 10)
-guard_true(i25, descr=<Guard0x7f551538c4d0>) [p1, p0, p11, p2, p3, p6, p13]
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #18 POP_JUMP_IF_FALSE')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #21 LOAD_CONST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #24 STORE_FAST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #27 LOAD_FAST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #30 LOAD_CONST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #33 INPLACE_ADD')
-+322: i27 = int_add(i23, 1)
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #34 STORE_FAST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #37 JUMP_ABSOLUTE')
-+327: guard_not_invalidated(descr=<Guard0x7f551538c458>) [p1, p0, p2, p3, p6, i27]
-+327: i29 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
-+335: i31 = int_lt(i29, 0)
-guard_false(i31, descr=<Guard0x7f551538c3e0>) [p1, p0, p2, p3, p6, i27]
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
-+345: label(p0, p1, p2, p3, p6, i27, descr=TargetToken(140003404595320))
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #12 LOAD_CONST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #15 COMPARE_OP')
-+376: i32 = int_lt(i27, 10)
-guard_true(i32, descr=<Guard0x7f551538c368>) [p1, p0, p2, p3, p6, i27]
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #18 POP_JUMP_IF_FALSE')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #21 LOAD_CONST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #24 STORE_FAST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #27 LOAD_FAST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #30 LOAD_CONST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #33 INPLACE_ADD')
-+386: i33 = int_add(i27, 1)
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #34 STORE_FAST')
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #37 JUMP_ABSOLUTE')
-+390: guard_not_invalidated(descr=<Guard0x7f551538c2f0>) [p1, p0, p2, p3, p6, i33, None]
-+390: i35 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
-+398: i36 = int_lt(i35, 0)
-guard_false(i36, descr=<Guard0x7f551538c278>) [p1, p0, p2, p3, p6, i33, None]
-debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
-+408: jump(p0, p1, p2, p3, p6, i33, descr=TargetToken(140003404595320))
-+416: --end of the loop--
-[1cffd9ba83b9] jit-log-opt-loop}
-[1cffd9d7af1e] {jit-backend
-[1cffd9ea4873] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b32128 +0  4881EC9800000048896C24604889FD48895C24684C896424704C896C24784C89B424800000004C89BC2488000000488B0425B07916034829E0483B0425109F0103760D49BBD61BB314557F000041FFD349BBD8509317557F00004D8B3B4D8D770149BBD8509317557F00004D89334C8BB5380100004D8BBE800000004D8B6E504D8B66704D0FB6968E0000004D8B4E604D8B4678498B7E58498B7668488B5E10488B5618488B4620488B4E284C89BD480100004C89AD500100004C898D580100004889BD6001000048899D6801000048898D7001000049BB38519317557F0000498B0B488D590149BB38519317557F000049891B4983F8030F85000000008138E82200000F85000000004C8B40104D85C00F8400000000488B5808498B48108139685505000F8500000000498B48084C8B4108488B79104C8B49184883FB000F8C000000004C39CB0F8D000000004889D9480FAFDF4D89C54901D8488D5901488958084983FA000F85000000004C8B521041813A089203000F85000000004C8B5208498B4A084C8D79014C8985600100004C89A56801000048898D700100004C898D78010000488985800100004C8995880100004889BD90010000488995980100004C89D74C89FE49BB8820B314557F00004C895D2041BBE060730041FFD3F6450401740D49BBFD11B314557F000041FFD348C745200000000048833C2500E61503000F8500000000488B9588010000488B7A104C8B9560010000488B85700100004C8954C710488B0425807816034883F8000F8C00000000488B856801000049BB584C3815557F00004C39D80F850000000049BB50519317557F00004D8B13498D420149BB50519317557F0000498903483B9D780100000F8D000000004889D8480FAF9D900100004D89EA4901DD488D5801488B4208488D780148899560010000488985680100004C8995700100004889FE4889D749BBE820B314557F00004C895D2041BBE060730041FFD3F6450401740D49BBFD11B314557F000041FFD348C74520000000004C8B958001000049895A0848833C2500E61503000F8500000000488B8560010000488B5010488BBD680100004C896CFA10488B3C25807816034883FF000F8C000000004C89AD600100004C8995800100004C8BAD700100004889C2E90BFFFFFF49BBA0967F17557F0000415349BB0820B314557F0000415349BB0010B314557F000041FFE349BB704A8317557F0000415349BB1820B314557F0000415349BB0010B314557F000041FFE349BBF8498317557F0000415349BB2820B314557F0000415349BB0010B314557F000041FFE349BB80498317557F0000415349BB3820B314557F0000415349BB0010B314557F000041FFE349BB08498317557F0000415349BB4820B314557F0000415349BB0010B314557F000041FFE349BB90488317557F0000415349BB5820B314557F0000415349BB0010B314557F000041FFE349BB18488317557F0000415349BB6820B314557F0000415349BB0010B314557F000041FFE349BBA0478317557F0000415349BB7820B314557F0000415349BB0010B314557F000041FFE349BB28478317557F0000415349BB9820B314557F0000415349BB8510B314557F000041FFE349BBB0468317557F0000415349BBA820B314557F0000415349BB0010B314557F000041FFE349BB38468317557F0000415349BBB820B314557F0000415349BB0010B314557F000041FFE349BBC0458317557F0000415349BBC820B314557F0000415349BB0010B314557F000041FFE349BB48458317557F0000415349BBD820B314557F0000415349BB0010B314557F000041FFE349BBD0448317557F0000415349BBF820B314557F0000415349BB8510B314557F000041FFE349BB58448317557F0000415349BB0821B314557F0000415349BB0010B314557F000041FFE349BBE0438317557F0000415349BB1821B314557F0000415349BB0010B314557F000041FFE3
-[1cffd9ebc29f] jit-backend-dump}
-[1cffd9ebcab0] {jit-backend-addr
-Loop 1 (<code object h, file 'x.py', line 14> #13 FOR_ITER) has address 0x7f5514b32178 to 0x7f5514b32470 (bootstrap 0x7f5514b32128)
-[1cffd9ebde77] jit-backend-addr}
-[1cffd9ebe969] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b32222 +0  4A020000
-[1cffd9ebfa23] jit-backend-dump}
-[1cffd9ec0059] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b3222e +0  63020000
-[1cffd9ec0ae6] jit-backend-dump}
-[1cffd9ec0f4f] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b3223b +0  7B020000
-[1cffd9ec18bc] jit-backend-dump}
-[1cffd9ec1d28] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b3224f +0  8C020000
-[1cffd9ec2689] jit-backend-dump}
-[1cffd9ec2b07] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b32269 +0  97020000
-[1cffd9ec3474] jit-backend-dump}
-[1cffd9ec38bc] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b32272 +0  B3020000
-[1cffd9ec4220] jit-backend-dump}
-[1cffd9ec4677] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b32291 +0  B9020000
-[1cffd9ec5011] jit-backend-dump}
-[1cffd9ec5459] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b322a2 +0  CD020000
-[1cffd9ec5e1c] jit-backend-dump}
-[1cffd9ec6279] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b3232d +0  67020000
-[1cffd9ec6bd7] jit-backend-dump}
-[1cffd9ec77c9] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b3235d +0  81020000
-[1cffd9ec8142] jit-backend-dump}
-[1cffd9ec85c0] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b32377 +0  8C020000
-[1cffd9ecbf8d] jit-backend-dump}
-[1cffd9ecc51d] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b323a2 +0  86020000
-[1cffd9eccee0] jit-backend-dump}
-[1cffd9ecd33a] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b32426 +0  27020000
-[1cffd9ecdc8f] jit-backend-dump}
-[1cffd9ece160] {jit-backend-dump
-BACKEND x86_64
-SYS_EXECUTABLE python
-CODE_DUMP @7f5514b3244f +0  48020000
-[1cffd9eceab8] jit-backend-dump}
-[1cffd9ecf545] jit-backend}
-[1cffd9ed0c35] {jit-log-opt-loop
-# Loop 1 (<code object h, file 'x.py', line 14> #13 FOR_ITER) : loop with 82 ops
-[p0, p1]
-+110: p2 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_w_f_trace 128>)
-+124: p3 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_last_exception 80>)
-+128: p4 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_pycode 112>)
-+132: i5 = getfield_gc(p0, descr=<FieldU pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 142>)
-+140: p6 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_lastblock 96>)
-+144: i7 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_valuestackdepth 120>)
-+148: i8 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_last_instr 88>)
-+152: p9 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_locals_stack_w 104>)
-+156: p11 = getarrayitem_gc(p9, 0, descr=<ArrayP 8>)
-+160: p13 = getarrayitem_gc(p9, 1, descr=<ArrayP 8>)
-+164: p15 = getarrayitem_gc(p9, 2, descr=<ArrayP 8>)
-+168: p17 = getarrayitem_gc(p9, 3, descr=<ArrayP 8>)
-+172: p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_cells 40>)
-+172: label(p0, p1, p2, p3, p4, i5, p6, i7, i8, p11, p13, p15, p17, descr=TargetToken(140003443320224))
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
-+244: guard_value(i7, 3, descr=<Guard0x7f55177f96a0>) [i7, p1, p0, p2, p3, p4, i5, p6, i8, p11, p13, p15, p17]
-+254: guard_class(p15, 26177128, descr=<Guard0x7f5517834a70>) [p1, p0, p15, p2, p3, p4, i5, p6, p11, p13, p17]
-+266: p21 = getfield_gc(p15, descr=<FieldP pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_w_seq 16>)
-+270: guard_nonnull(p21, descr=<Guard0x7f55178349f8>) [p1, p0, p15, p21, p2, p3, p4, i5, p6, p11, p13, p17]
-+279: i22 = getfield_gc(p15, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
-+283: p23 = getfield_gc(p21, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_strategy 16>)
-+287: guard_class(p23, 26517736, descr=<Guard0x7f5517834980>) [p1, p0, p15, i22, p23, p21, p2, p3, p4, i5, p6, p11, p13, p17]
-+299: p25 = getfield_gc(p21, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_lstorage 8>)
-+303: i26 = getfield_gc_pure(p25, descr=<FieldS tuple3.item0 8>)
-+307: i27 = getfield_gc_pure(p25, descr=<FieldS tuple3.item1 16>)
-+311: i28 = getfield_gc_pure(p25, descr=<FieldS tuple3.item2 24>)
-+315: i30 = int_lt(i22, 0)
-guard_false(i30, descr=<Guard0x7f5517834908>) [p1, p0, p15, i22, i28, i27, i26, p2, p3, p4, i5, p6, p11, p13, p17]
-+325: i31 = int_ge(i22, i28)
-guard_false(i31, descr=<Guard0x7f5517834890>) [p1, p0, p15, i22, i27, i26, p2, p3, p4, i5, p6, p11, p13, p17]
-+334: i32 = int_mul(i22, i27)
-+341: i33 = int_add(i26, i32)
-+347: i35 = int_add(i22, 1)
-+351: setfield_gc(p15, i35, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
-+355: guard_value(i5, 0, descr=<Guard0x7f5517834818>) [i5, p1, p0, p2, p3, p4, p6, p11, p13, p15, i33]
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #16 STORE_FAST')
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #19 LOAD_FAST')
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #22 LIST_APPEND')
-+365: p37 = getfield_gc(p13, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_strategy 16>)
-+369: guard_class(p37, 26402184, descr=<Guard0x7f55178347a0>) [p1, p0, p37, p13, p2, p3, p4, p6, p15, i33]
-+382: p39 = getfield_gc(p13, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_lstorage 8>)
-+386: i40 = getfield_gc(p39, descr=<FieldS list.length 8>)
-+390: i42 = int_add(i40, 1)
-+394: p43 = getfield_gc(p39, descr=<FieldP list.items 16>)
-+394: i44 = arraylen_gc(p43, descr=<ArrayS 8>)
-+394: call(ConstClass(_ll_list_resize_ge_trampoline__v672___simple_call__function__), p39, i42, descr=<Callv 0 ri EF=4>)
-+506: guard_no_exception(descr=<Guard0x7f5517834728>) [p1, p0, i40, i33, p39, p2, p3, p4, p6, p13, p15, None]
-+521: p47 = getfield_gc(p39, descr=<FieldP list.items 16>)
-+532: setarrayitem_gc(p47, i40, i33, descr=<ArrayS 8>)
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #25 JUMP_ABSOLUTE')
-+551: guard_not_invalidated(descr=<Guard0x7f55178346b0>) [p1, p0, p2, p3, p4, p6, p13, p15, i33]
-+551: i49 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
-+559: i51 = int_lt(i49, 0)
-guard_false(i51, descr=<Guard0x7f5517834638>) [p1, p0, p2, p3, p4, p6, p13, p15, i33]
-+569: guard_value(p4, ConstPtr(ptr52), descr=<Guard0x7f55178345c0>) [p1, p0, p4, p2, p3, p6, p13, p15, i33]
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
-+595: label(p0, p1, p2, p3, p6, i33, p13, p15, i35, i28, i27, i26, p39, descr=TargetToken(140003443320312))
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
-+625: i53 = int_ge(i35, i28)
-guard_false(i53, descr=<Guard0x7f5517834548>) [p1, p0, p15, i35, i27, i26, p2, p3, p6, p13, i33]
-+638: i54 = int_mul(i35, i27)
-+649: i55 = int_add(i26, i54)
-+655: i56 = int_add(i35, 1)
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #16 STORE_FAST')
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #19 LOAD_FAST')
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #22 LIST_APPEND')
-+659: i57 = getfield_gc(p39, descr=<FieldS list.length 8>)
-+663: i58 = int_add(i57, 1)
-+667: p59 = getfield_gc(p39, descr=<FieldP list.items 16>)
-+667: i60 = arraylen_gc(p59, descr=<ArrayS 8>)
-+667: call(ConstClass(_ll_list_resize_ge_trampoline__v672___simple_call__function__), p39, i58, descr=<Callv 0 ri EF=4>)
-+744: setfield_gc(p15, i56, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
-+755: guard_no_exception(descr=<Guard0x7f55178344d0>) [p1, p0, i57, i55, p39, p2, p3, p6, p13, p15, None]
-+770: p61 = getfield_gc(p39, descr=<FieldP list.items 16>)
-+781: setarrayitem_gc(p61, i57, i55, descr=<ArrayS 8>)
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #25 JUMP_ABSOLUTE')
-+793: guard_not_invalidated(descr=<Guard0x7f5517834458>) [p1, p0, p2, p3, p6, p13, p15, i55, None]
-+793: i62 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
-+801: i63 = int_lt(i62, 0)
-guard_false(i63, descr=<Guard0x7f55178343e0>) [p1, p0, p2, p3, p6, p13, p15, i55, None]
-debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
-+811: jump(p0, p1, p2, p3, p6, i55, p13, p15, i56, i28, i27, i26, p39, descr=TargetToken(140003443320312))
-+840: --end of the loop--
-[1cffd9f27224] jit-log-opt-loop}
-[1cffd9f6f244] {jit-backend-counts
-entry 0:1
-TargetToken(140003404595232):1
-TargetToken(140003404595320):4
-entry 1:1
-TargetToken(140003443320224):1
-TargetToken(140003443320312):4
-[1cffd9f72430] jit-backend-counts}
+0# [1cffd8feb691] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31000 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd8ffaba6] jit-backend-dump}
+0# [1cffd90012ee] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31085 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000488B1C2508E6150348C7042500E615030000000048C7042508E615030000000048895D38584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd9003b76] jit-backend-dump}
+0# [1cffd900719f] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3112e +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000488B7C240841BBB064120141FFD3488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000C20800
+0# [1cffd9008c81] jit-backend-dump}
+0# [1cffd900b384] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31191 +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000488B7C240841BBF065120141FFD3488B442408F6400480488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000C20800
+0# [1cffd900cf18] jit-backend-dump}
+0# [1cffd9010345] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b311fd +0  4883EC384889442408F20F114424184889EF48895C24284C89642430488B1C2508E615034C8B242500E6150348C7042500E615030000000048C7042508E615030000000041BBB064120141FFD3F20F10442418488B44240848891C2508E615034C89242500E61503488B5C24284C8B642430488D642438C3
+0# [1cffd9011f0b] jit-backend-dump}
+0# [1cffd9015bd8] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31275 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B4C240848894D20488B7424104889EF4883EC0848C7452810000000488B0C2508E6150348894D38488B1C2500E6150348C7042500E615030000000048C7042508E615030000000041BB60DBE80041FFD34889C5488B4D3848C745380000000048890C2508E6150348891C2500E615034883C40848C745280000000048C7452000000000488B4D58488B4560488B5568488B5D70488B7578488BBD800000004C8B85880000004C8B8D900000004C8B95980000004C8BA5A00000004C8BADA80000004C8BB5B00000004C8BBDB8000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C3
+0# [1cffd901a191] jit-backend-dump}
+0# [1cffd901b3a6] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31491 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd901dc46] jit-backend-dump}
+0# [1cffd901ef79] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31595 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B1C2508E6150348C7042500E615030000000048C7042508E615030000000048895D38584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd902ce01] jit-backend-dump}
+0# [1cffd902e819] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b316bd +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B7C240841BBB064120141FFD3488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C20800
+0# [1cffd9031b79] jit-backend-dump}
+0# [1cffd90331b0] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3181e +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B7C240841BBF065120141FFD3488B442408F6400480488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C20800
+0# [1cffd903629a] jit-backend-dump}
+0# [1cffd903736b] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31988 +0  488B042508E6150348C7042500E615030000000048C7042508E61503000000004889453848C7451000C2B5014889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd9038a70] jit-backend-dump}
+0# [1cffd903e2cd] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b319e3 +0  48894D584889556848895D70488975784C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B4C240848894D204829C74883EC0848C745281000000041BBB0A4E80041FFD34883C4084885C00F84F4000000F645040174154883EC0849BBFD11B314557F000041FFD34883C40848C7452800000000488B4D58488B5568488B5D70488B75784C8B85880000004C8B8D900000004C8B95980000004C8BA5A00000004C8BADA80000004C8BB5B00000004C8BBDB8000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000488B3C25484CB60148C7452000000000C34883C40849BB8819B314557F000041FFE3
+0# [1cffd904265b] jit-backend-dump}
+0# [1cffd90448f2] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31bd6 +0  4889E74883EC0841BBD00F210141FFD34883C408488B042500E615034885C07501C34883C40849BB8819B314557F000041FFE3
+0# [1cffd9045d15] jit-backend-dump}
+0# [1cffd904647a] {jit-backend-counts
+0# [1cffd9046851] jit-backend-counts}
+0# [1cffd9636773] {jit-backend
+0# [1cffd9afbdde] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31ce0 +0  4881EC9800000048896C24604889FD48895C24684C896424704C896C24784C89B424800000004C89BC2488000000488B0425B07916034829E0483B0425109F0103760D49BBD61BB314557F000041FFD349BBF0509317557F00004D8B3B4D8D770149BBF0509317557F00004D89334C8BB5380100004D8BBE800000004D8B6E504D8B66704D0FB6968E0000004D8B4E604D8B4678498B7E58498B7668488B5E10488B5618488B4620488B4E284C89BD480100004C89AD500100004C898D580100004889BD60010000488995680100004889857001000048898D7801000049BB08519317557F0000498B0B488D410149BB08519317557F00004989034983F8020F85000000004883FB017206813B180C00000F85000000004983FA000F850000000049BB20803215557F00004D39DC0F85000000004C8B63084983FC0A0F8D00000000498D5C24014C8B2425807816034983FC000F8C0000000049BB20519317557F00004D8B234D8D54240149BB20519317557F00004D89134883FB0A0F8D000000004C8D5301488B1C25807816034883FB000F8C000000004C89D3E9B9FFFFFF49BB20E03215557F0000415349BB401CB314557F0000415349BB0010B314557F000041FFE349BB38C63815557F0000415349BB501CB314557F0000415349BB0010B314557F000041FFE349BBC0C53815557F0000415349BB601CB314557F0000415349BB0010B314557F000041FFE349BB48C53815557F0000415349BB701CB314557F0000415349BB0010B314557F000041FFE349BBD0C43815557F0000415349BB801CB314557F0000415349BB0010B314557F000041FFE349BB58C43815557F0000415349BB901CB314557F0000415349BB0010B314557F000041FFE349BBE0C33815557F0000415349BBA01CB314557F0000415349BB0010B314557F000041FFE349BB68C33815557F0000415349BBB01CB314557F0000415349BB0010B314557F000041FFE349BBF0C23815557F0000415349BBC01CB314557F0000415349BB0010B314557F000041FFE349BB78C23815557F0000415349BBD01CB314557F0000415349BB0010B314557F000041FFE3
+0# [1cffd9b146d6] jit-backend-dump}
+0# [1cffd9b14ff3] {jit-backend-addr
+0# Loop 0 (<code object g, file 'x.py', line 8> #9 LOAD_FAST) has address 0x7f5514b31d30 to 0x7f5514b31e80 (bootstrap 0x7f5514b31ce0)
+0# [1cffd9b16753] jit-backend-addr}
+0# [1cffd9b17245] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31de1 +0  9B000000
+0# [1cffd9b18103] jit-backend-dump}
+0# [1cffd9b18762] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31df3 +0  AE000000
+0# [1cffd9b191ae] jit-backend-dump}
+0# [1cffd9b1960b] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31dfd +0  C9000000
+0# [1cffd9b19f1f] jit-backend-dump}
+0# [1cffd9b1a32f] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e10 +0  DB000000
+0# [1cffd9b1ac8d] jit-backend-dump}
+0# [1cffd9b1b091] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e1e +0  F2000000
+0# [1cffd9b1ba54] jit-backend-dump}
+0# [1cffd9b1bfec] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e35 +0  25010000
+0# [1cffd9b1c8d3] jit-backend-dump}
+0# [1cffd9b1ccfb] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e5e +0  21010000
+0# [1cffd9b1d5d6] jit-backend-dump}
+0# [1cffd9b1da25] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e74 +0  55010000
+0# [1cffd9b1e3b8] jit-backend-dump}
+0# [1cffd9b1f0b2] jit-backend}
+0# [1cffd9b20d9f] {jit-log-opt-loop
+0# # Loop 0 (<code object g, file 'x.py', line 8> #9 LOAD_FAST) : loop with 59 ops
+0# [p0, p1]
+0# +110: p2 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_w_f_trace 128>)
+0# +124: p3 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_last_exception 80>)
+0# +128: p4 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_pycode 112>)
+0# +132: i5 = getfield_gc(p0, descr=<FieldU pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 142>)
+0# +140: p6 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_lastblock 96>)
+0# +144: i7 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_valuestackdepth 120>)
+0# +148: i8 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_last_instr 88>)
+0# +152: p9 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_locals_stack_w 104>)
+0# +156: p11 = getarrayitem_gc(p9, 0, descr=<ArrayP 8>)
+0# +160: p13 = getarrayitem_gc(p9, 1, descr=<ArrayP 8>)
+0# +164: p15 = getarrayitem_gc(p9, 2, descr=<ArrayP 8>)
+0# +168: p17 = getarrayitem_gc(p9, 3, descr=<ArrayP 8>)
+0# +172: p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_cells 40>)
+0# +172: label(p0, p1, p2, p3, p4, i5, p6, i7, i8, p11, p13, p15, p17, descr=TargetToken(140003404595232))
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# +251: guard_value(i7, 2, descr=<Guard0x7f551532e020>) [i7, p1, p0, p2, p3, p4, i5, p6, i8, p11, p13, p15, p17]
+0# +261: guard_nonnull_class(p11, ConstClass(W_IntObject), descr=<Guard0x7f551538c638>) [p1, p0, p11, p2, p3, p4, i5, p6, p13, p15, p17]
+0# +279: guard_value(i5, 0, descr=<Guard0x7f551538c5c0>) [i5, p1, p0, p2, p3, p4, p6, p11, p13, p17]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #12 LOAD_CONST')
+0# +289: guard_value(p4, ConstPtr(ptr22), descr=<Guard0x7f551538c548>) [p1, p0, p4, p2, p3, p6, p11, p13, p17]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #15 COMPARE_OP')
+0# +308: i23 = getfield_gc_pure(p11, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval 8>)
+0# +312: i25 = int_lt(i23, 10)
+0# guard_true(i25, descr=<Guard0x7f551538c4d0>) [p1, p0, p11, p2, p3, p6, p13]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #18 POP_JUMP_IF_FALSE')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #21 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #24 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #27 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #30 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #33 INPLACE_ADD')
+0# +322: i27 = int_add(i23, 1)
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #34 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #37 JUMP_ABSOLUTE')
+0# +327: guard_not_invalidated(descr=<Guard0x7f551538c458>) [p1, p0, p2, p3, p6, i27]
+0# +327: i29 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +335: i31 = int_lt(i29, 0)
+0# guard_false(i31, descr=<Guard0x7f551538c3e0>) [p1, p0, p2, p3, p6, i27]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# +345: label(p0, p1, p2, p3, p6, i27, descr=TargetToken(140003404595320))
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #12 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #15 COMPARE_OP')
+0# +376: i32 = int_lt(i27, 10)
+0# guard_true(i32, descr=<Guard0x7f551538c368>) [p1, p0, p2, p3, p6, i27]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #18 POP_JUMP_IF_FALSE')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #21 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #24 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #27 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #30 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #33 INPLACE_ADD')
+0# +386: i33 = int_add(i27, 1)
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #34 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #37 JUMP_ABSOLUTE')
+0# +390: guard_not_invalidated(descr=<Guard0x7f551538c2f0>) [p1, p0, p2, p3, p6, i33, None]
+0# +390: i35 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +398: i36 = int_lt(i35, 0)
+0# guard_false(i36, descr=<Guard0x7f551538c278>) [p1, p0, p2, p3, p6, i33, None]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# +408: jump(p0, p1, p2, p3, p6, i33, descr=TargetToken(140003404595320))
+0# +416: --end of the loop--
+0# [1cffd9ba83b9] jit-log-opt-loop}
+0# [1cffd9d7af1e] {jit-backend
+0# [1cffd9ea4873] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32128 +0  4881EC9800000048896C24604889FD48895C24684C896424704C896C24784C89B424800000004C89BC2488000000488B0425B07916034829E0483B0425109F0103760D49BBD61BB314557F000041FFD349BBD8509317557F00004D8B3B4D8D770149BBD8509317557F00004D89334C8BB5380100004D8BBE800000004D8B6E504D8B66704D0FB6968E0000004D8B4E604D8B4678498B7E58498B7668488B5E10488B5618488B4620488B4E284C89BD480100004C89AD500100004C898D580100004889BD6001000048899D6801000048898D7001000049BB38519317557F0000498B0B488D590149BB38519317557F000049891B4983F8030F85000000008138E82200000F85000000004C8B40104D85C00F8400000000488B5808498B48108139685505000F8500000000498B48084C8B4108488B79104C8B49184883FB000F8C000000004C39CB0F8D000000004889D9480FAFDF4D89C54901D8488D5901488958084983FA000F85000000004C8B521041813A089203000F85000000004C8B5208498B4A084C8D79014C8985600100004C89A56801000048898D700100004C898D78010000488985800100004C8995880100004889BD90010000488995980100004C89D74C89FE49BB8820B314557F00004C895D2041BBE060730041FFD3F6450401740D49BBFD11B314557F000041FFD348C745200000000048833C2500E61503000F8500000000488B9588010000488B7A104C8B9560010000488B85700100004C8954C710488B0425807816034883F8000F8C00000000488B856801000049BB584C3815557F00004C39D80F850000000049BB50519317557F00004D8B13498D420149BB50519317557F0000498903483B9D780100000F8D000000004889D8480FAF9D900100004D89EA4901DD488D5801488B4208488D780148899560010000488985680100004C8995700100004889FE4889D749BBE820B314557F00004C895D2041BBE060730041FFD3F6450401740D49BBFD11B314557F000041FFD348C74520000000004C8B958001000049895A0848833C2500E61503000F8500000000488B8560010000488B5010488BBD680100004C896CFA10488B3C25807816034883FF000F8C000000004C89AD600100004C8995800100004C8BAD700100004889C2E90BFFFFFF49BBA0967F17557F0000415349BB0820B314557F0000415349BB0010B314557F000041FFE349BB704A8317557F0000415349BB1820B314557F0000415349BB0010B314557F000041FFE349BBF8498317557F0000415349BB2820B314557F0000415349BB0010B314557F000041FFE349BB80498317557F0000415349BB3820B314557F0000415349BB0010B314557F000041FFE349BB08498317557F0000415349BB4820B314557F0000415349BB0010B314557F000041FFE349BB90488317557F0000415349BB5820B314557F0000415349BB0010B314557F000041FFE349BB18488317557F0000415349BB6820B314557F0000415349BB0010B314557F000041FFE349BBA0478317557F0000415349BB7820B314557F0000415349BB0010B314557F000041FFE349BB28478317557F0000415349BB9820B314557F0000415349BB8510B314557F000041FFE349BBB0468317557F0000415349BBA820B314557F0000415349BB0010B314557F000041FFE349BB38468317557F0000415349BBB820B314557F0000415349BB0010B314557F000041FFE349BBC0458317557F0000415349BBC820B314557F0000415349BB0010B314557F000041FFE349BB48458317557F0000415349BBD820B314557F0000415349BB0010B314557F000041FFE349BBD0448317557F0000415349BBF820B314557F0000415349BB8510B314557F000041FFE349BB58448317557F0000415349BB0821B314557F0000415349BB0010B314557F000041FFE349BBE0438317557F0000415349BB1821B314557F0000415349BB0010B314557F000041FFE3
+0# [1cffd9ebc29f] jit-backend-dump}
+0# [1cffd9ebcab0] {jit-backend-addr
+0# Loop 1 (<code object h, file 'x.py', line 14> #13 FOR_ITER) has address 0x7f5514b32178 to 0x7f5514b32470 (bootstrap 0x7f5514b32128)
+0# [1cffd9ebde77] jit-backend-addr}
+0# [1cffd9ebe969] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32222 +0  4A020000
+0# [1cffd9ebfa23] jit-backend-dump}
+0# [1cffd9ec0059] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3222e +0  63020000
+0# [1cffd9ec0ae6] jit-backend-dump}
+0# [1cffd9ec0f4f] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3223b +0  7B020000
+0# [1cffd9ec18bc] jit-backend-dump}
+0# [1cffd9ec1d28] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3224f +0  8C020000
+0# [1cffd9ec2689] jit-backend-dump}
+0# [1cffd9ec2b07] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32269 +0  97020000
+0# [1cffd9ec3474] jit-backend-dump}
+0# [1cffd9ec38bc] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32272 +0  B3020000
+0# [1cffd9ec4220] jit-backend-dump}
+0# [1cffd9ec4677] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32291 +0  B9020000
+0# [1cffd9ec5011] jit-backend-dump}
+0# [1cffd9ec5459] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b322a2 +0  CD020000
+0# [1cffd9ec5e1c] jit-backend-dump}
+0# [1cffd9ec6279] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3232d +0  67020000
+0# [1cffd9ec6bd7] jit-backend-dump}
+0# [1cffd9ec77c9] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3235d +0  81020000
+0# [1cffd9ec8142] jit-backend-dump}
+0# [1cffd9ec85c0] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32377 +0  8C020000
+0# [1cffd9ecbf8d] jit-backend-dump}
+0# [1cffd9ecc51d] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b323a2 +0  86020000
+0# [1cffd9eccee0] jit-backend-dump}
+0# [1cffd9ecd33a] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32426 +0  27020000
+0# [1cffd9ecdc8f] jit-backend-dump}
+0# [1cffd9ece160] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3244f +0  48020000
+0# [1cffd9eceab8] jit-backend-dump}
+0# [1cffd9ecf545] jit-backend}
+0# [1cffd9ed0c35] {jit-log-opt-loop
+0# # Loop 1 (<code object h, file 'x.py', line 14> #13 FOR_ITER) : loop with 82 ops
+0# [p0, p1]
+0# +110: p2 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_w_f_trace 128>)
+0# +124: p3 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_last_exception 80>)
+0# +128: p4 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_pycode 112>)
+0# +132: i5 = getfield_gc(p0, descr=<FieldU pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 142>)
+0# +140: p6 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_lastblock 96>)
+0# +144: i7 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_valuestackdepth 120>)
+0# +148: i8 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_last_instr 88>)
+0# +152: p9 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_locals_stack_w 104>)
+0# +156: p11 = getarrayitem_gc(p9, 0, descr=<ArrayP 8>)
+0# +160: p13 = getarrayitem_gc(p9, 1, descr=<ArrayP 8>)
+0# +164: p15 = getarrayitem_gc(p9, 2, descr=<ArrayP 8>)
+0# +168: p17 = getarrayitem_gc(p9, 3, descr=<ArrayP 8>)
+0# +172: p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_cells 40>)
+0# +172: label(p0, p1, p2, p3, p4, i5, p6, i7, i8, p11, p13, p15, p17, descr=TargetToken(140003443320224))
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +244: guard_value(i7, 3, descr=<Guard0x7f55177f96a0>) [i7, p1, p0, p2, p3, p4, i5, p6, i8, p11, p13, p15, p17]
+0# +254: guard_class(p15, 26177128, descr=<Guard0x7f5517834a70>) [p1, p0, p15, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +266: p21 = getfield_gc(p15, descr=<FieldP pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_w_seq 16>)
+0# +270: guard_nonnull(p21, descr=<Guard0x7f55178349f8>) [p1, p0, p15, p21, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +279: i22 = getfield_gc(p15, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
+0# +283: p23 = getfield_gc(p21, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_strategy 16>)
+0# +287: guard_class(p23, 26517736, descr=<Guard0x7f5517834980>) [p1, p0, p15, i22, p23, p21, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +299: p25 = getfield_gc(p21, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_lstorage 8>)
+0# +303: i26 = getfield_gc_pure(p25, descr=<FieldS tuple3.item0 8>)
+0# +307: i27 = getfield_gc_pure(p25, descr=<FieldS tuple3.item1 16>)
+0# +311: i28 = getfield_gc_pure(p25, descr=<FieldS tuple3.item2 24>)
+0# +315: i30 = int_lt(i22, 0)
+0# guard_false(i30, descr=<Guard0x7f5517834908>) [p1, p0, p15, i22, i28, i27, i26, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +325: i31 = int_ge(i22, i28)
+0# guard_false(i31, descr=<Guard0x7f5517834890>) [p1, p0, p15, i22, i27, i26, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +334: i32 = int_mul(i22, i27)
+0# +341: i33 = int_add(i26, i32)
+0# +347: i35 = int_add(i22, 1)
+0# +351: setfield_gc(p15, i35, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
+0# +355: guard_value(i5, 0, descr=<Guard0x7f5517834818>) [i5, p1, p0, p2, p3, p4, p6, p11, p13, p15, i33]
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #16 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #19 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #22 LIST_APPEND')
+0# +365: p37 = getfield_gc(p13, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_strategy 16>)
+0# +369: guard_class(p37, 26402184, descr=<Guard0x7f55178347a0>) [p1, p0, p37, p13, p2, p3, p4, p6, p15, i33]
+0# +382: p39 = getfield_gc(p13, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_lstorage 8>)
+0# +386: i40 = getfield_gc(p39, descr=<FieldS list.length 8>)
+0# +390: i42 = int_add(i40, 1)
+0# +394: p43 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +394: i44 = arraylen_gc(p43, descr=<ArrayS 8>)
+0# +394: call(ConstClass(_ll_list_resize_ge_trampoline__v672___simple_call__function__), p39, i42, descr=<Callv 0 ri EF=4>)
+0# +506: guard_no_exception(descr=<Guard0x7f5517834728>) [p1, p0, i40, i33, p39, p2, p3, p4, p6, p13, p15, None]
+0# +521: p47 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +532: setarrayitem_gc(p47, i40, i33, descr=<ArrayS 8>)
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #25 JUMP_ABSOLUTE')
+0# +551: guard_not_invalidated(descr=<Guard0x7f55178346b0>) [p1, p0, p2, p3, p4, p6, p13, p15, i33]
+0# +551: i49 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +559: i51 = int_lt(i49, 0)
+0# guard_false(i51, descr=<Guard0x7f5517834638>) [p1, p0, p2, p3, p4, p6, p13, p15, i33]
+0# +569: guard_value(p4, ConstPtr(ptr52), descr=<Guard0x7f55178345c0>) [p1, p0, p4, p2, p3, p6, p13, p15, i33]
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +595: label(p0, p1, p2, p3, p6, i33, p13, p15, i35, i28, i27, i26, p39, descr=TargetToken(140003443320312))
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +625: i53 = int_ge(i35, i28)
+0# guard_false(i53, descr=<Guard0x7f5517834548>) [p1, p0, p15, i35, i27, i26, p2, p3, p6, p13, i33]
+0# +638: i54 = int_mul(i35, i27)
+0# +649: i55 = int_add(i26, i54)
+0# +655: i56 = int_add(i35, 1)
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #16 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #19 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #22 LIST_APPEND')
+0# +659: i57 = getfield_gc(p39, descr=<FieldS list.length 8>)
+0# +663: i58 = int_add(i57, 1)
+0# +667: p59 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +667: i60 = arraylen_gc(p59, descr=<ArrayS 8>)
+0# +667: call(ConstClass(_ll_list_resize_ge_trampoline__v672___simple_call__function__), p39, i58, descr=<Callv 0 ri EF=4>)
+0# +744: setfield_gc(p15, i56, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
+0# +755: guard_no_exception(descr=<Guard0x7f55178344d0>) [p1, p0, i57, i55, p39, p2, p3, p6, p13, p15, None]
+0# +770: p61 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +781: setarrayitem_gc(p61, i57, i55, descr=<ArrayS 8>)
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #25 JUMP_ABSOLUTE')
+0# +793: guard_not_invalidated(descr=<Guard0x7f5517834458>) [p1, p0, p2, p3, p6, p13, p15, i55, None]
+0# +793: i62 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +801: i63 = int_lt(i62, 0)
+0# guard_false(i63, descr=<Guard0x7f55178343e0>) [p1, p0, p2, p3, p6, p13, p15, i55, None]
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +811: jump(p0, p1, p2, p3, p6, i55, p13, p15, i56, i28, i27, i26, p39, descr=TargetToken(140003443320312))
+0# +840: --end of the loop--
+0# [1cffd9f27224] jit-log-opt-loop}
+0# [1cffd9f6f244] {jit-backend-counts
+0# entry 0:1
+0# TargetToken(140003404595232):1
+0# TargetToken(140003404595320):4
+0# entry 1:1
+0# TargetToken(140003443320224):1
+0# TargetToken(140003443320312):4
+0# [1cffd9f72430] jit-backend-counts}
\ No newline at end of file
diff --git a/pypy/tool/jitlogparser/test/logtest_noopt.log b/pypy/tool/jitlogparser/test/logtest_noopt.log
new file mode 100644
--- /dev/null
+++ b/pypy/tool/jitlogparser/test/logtest_noopt.log
@@ -0,0 +1,356 @@
+0# [1cffd8feb691] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31000 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd8ffaba6] jit-backend-dump}
+0# [1cffd90012ee] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31085 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000488B1C2508E6150348C7042500E615030000000048C7042508E615030000000048895D38584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd9003b76] jit-backend-dump}
+0# [1cffd900719f] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3112e +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000488B7C240841BBB064120141FFD3488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000C20800
+0# [1cffd9008c81] jit-backend-dump}
+0# [1cffd900b384] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31191 +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000488B7C240841BBF065120141FFD3488B442408F6400480488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000C20800
+0# [1cffd900cf18] jit-backend-dump}
+0# [1cffd9010345] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b311fd +0  4883EC384889442408F20F114424184889EF48895C24284C89642430488B1C2508E615034C8B242500E6150348C7042500E615030000000048C7042508E615030000000041BBB064120141FFD3F20F10442418488B44240848891C2508E615034C89242500E61503488B5C24284C8B642430488D642438C3
+0# [1cffd9011f0b] jit-backend-dump}
+0# [1cffd9015bd8] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31275 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B4C240848894D20488B7424104889EF4883EC0848C7452810000000488B0C2508E6150348894D38488B1C2500E6150348C7042500E615030000000048C7042508E615030000000041BB60DBE80041FFD34889C5488B4D3848C745380000000048890C2508E6150348891C2500E615034883C40848C745280000000048C7452000000000488B4D58488B4560488B5568488B5D70488B7578488BBD800000004C8B85880000004C8B8D900000004C8B95980000004C8BA5A00000004C8BADA80000004C8BB5B00000004C8BBDB8000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C3
+0# [1cffd901a191] jit-backend-dump}
+0# [1cffd901b3a6] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31491 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd901dc46] jit-backend-dump}
+0# [1cffd901ef79] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31595 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B1C2508E6150348C7042500E615030000000048C7042508E615030000000048895D38584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd902ce01] jit-backend-dump}
+0# [1cffd902e819] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b316bd +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B7C240841BBB064120141FFD3488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C20800
+0# [1cffd9031b79] jit-backend-dump}
+0# [1cffd90331b0] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3181e +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B7C240841BBF065120141FFD3488B442408F6400480488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C20800
+0# [1cffd903629a] jit-backend-dump}
+0# [1cffd903736b] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31988 +0  488B042508E6150348C7042500E615030000000048C7042508E61503000000004889453848C7451000C2B5014889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd9038a70] jit-backend-dump}
+0# [1cffd903e2cd] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b319e3 +0  48894D584889556848895D70488975784C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B4C240848894D204829C74883EC0848C745281000000041BBB0A4E80041FFD34883C4084885C00F84F4000000F645040174154883EC0849BBFD11B314557F000041FFD34883C40848C7452800000000488B4D58488B5568488B5D70488B75784C8B85880000004C8B8D900000004C8B95980000004C8BA5A00000004C8BADA80000004C8BB5B00000004C8BBDB8000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000488B3C25484CB60148C7452000000000C34883C40849BB8819B314557F000041FFE3
+0# [1cffd904265b] jit-backend-dump}
+0# [1cffd90448f2] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31bd6 +0  4889E74883EC0841BBD00F210141FFD34883C408488B042500E615034885C07501C34883C40849BB8819B314557F000041FFE3
+0# [1cffd9045d15] jit-backend-dump}
+0# [1cffd904647a] {jit-backend-counts
+0# [1cffd9046851] jit-backend-counts}
+0# [1cffd9636773] {jit-backend
+0# [1cffd9afbdde] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31ce0 +0  4881EC9800000048896C24604889FD48895C24684C896424704C896C24784C89B424800000004C89BC2488000000488B0425B07916034829E0483B0425109F0103760D49BBD61BB314557F000041FFD349BBF0509317557F00004D8B3B4D8D770149BBF0509317557F00004D89334C8BB5380100004D8BBE800000004D8B6E504D8B66704D0FB6968E0000004D8B4E604D8B4678498B7E58498B7668488B5E10488B5618488B4620488B4E284C89BD480100004C89AD500100004C898D580100004889BD60010000488995680100004889857001000048898D7801000049BB08519317557F0000498B0B488D410149BB08519317557F00004989034983F8020F85000000004883FB017206813B180C00000F85000000004983FA000F850000000049BB20803215557F00004D39DC0F85000000004C8B63084983FC0A0F8D00000000498D5C24014C8B2425807816034983FC000F8C0000000049BB20519317557F00004D8B234D8D54240149BB20519317557F00004D89134883FB0A0F8D000000004C8D5301488B1C25807816034883FB000F8C000000004C89D3E9B9FFFFFF49BB20E03215557F0000415349BB401CB314557F0000415349BB0010B314557F000041FFE349BB38C63815557F0000415349BB501CB314557F0000415349BB0010B314557F000041FFE349BBC0C53815557F0000415349BB601CB314557F0000415349BB0010B314557F000041FFE349BB48C53815557F0000415349BB701CB314557F0000415349BB0010B314557F000041FFE349BBD0C43815557F0000415349BB801CB314557F0000415349BB0010B314557F000041FFE349BB58C43815557F0000415349BB901CB314557F0000415349BB0010B314557F000041FFE349BBE0C33815557F0000415349BBA01CB314557F0000415349BB0010B314557F000041FFE349BB68C33815557F0000415349BBB01CB314557F0000415349BB0010B314557F000041FFE349BBF0C23815557F0000415349BBC01CB314557F0000415349BB0010B314557F000041FFE349BB78C23815557F0000415349BBD01CB314557F0000415349BB0010B314557F000041FFE3
+0# [1cffd9b146d6] jit-backend-dump}
+0# [1cffd9b14ff3] {jit-backend-addr
+0# Loop 0 (<code object g, file 'x.py', line 8> #9 LOAD_FAST) has address 0x7f5514b31d30 to 0x7f5514b31e80 (bootstrap 0x7f5514b31ce0)
+0# [1cffd9b16753] jit-backend-addr}
+0# [1cffd9b17245] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31de1 +0  9B000000
+0# [1cffd9b18103] jit-backend-dump}
+0# [1cffd9b18762] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31df3 +0  AE000000
+0# [1cffd9b191ae] jit-backend-dump}
+0# [1cffd9b1960b] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31dfd +0  C9000000
+0# [1cffd9b19f1f] jit-backend-dump}
+0# [1cffd9b1a32f] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e10 +0  DB000000
+0# [1cffd9b1ac8d] jit-backend-dump}
+0# [1cffd9b1b091] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e1e +0  F2000000
+0# [1cffd9b1ba54] jit-backend-dump}
+0# [1cffd9b1bfec] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e35 +0  25010000
+0# [1cffd9b1c8d3] jit-backend-dump}
+0# [1cffd9b1ccfb] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e5e +0  21010000
+0# [1cffd9b1d5d6] jit-backend-dump}
+0# [1cffd9b1da25] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e74 +0  55010000
+0# [1cffd9b1e3b8] jit-backend-dump}
+0# [1cffd9b1f0b2] jit-backend}
+0# [1cffd9b20d9f] {jit-log-noopt-loop
+0# # Loop 0 (<code object g, file 'x.py', line 8> #9 LOAD_FAST) : loop with 59 ops
+0# [p0, p1]
+0# +110: p2 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_w_f_trace 128>)
+0# +124: p3 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_last_exception 80>)
+0# +128: p4 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_pycode 112>)
+0# +132: i5 = getfield_gc(p0, descr=<FieldU pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 142>)
+0# +140: p6 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_lastblock 96>)
+0# +144: i7 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_valuestackdepth 120>)
+0# +148: i8 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_last_instr 88>)
+0# +152: p9 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_locals_stack_w 104>)
+0# +156: p11 = getarrayitem_gc(p9, 0, descr=<ArrayP 8>)
+0# +160: p13 = getarrayitem_gc(p9, 1, descr=<ArrayP 8>)
+0# +164: p15 = getarrayitem_gc(p9, 2, descr=<ArrayP 8>)
+0# +168: p17 = getarrayitem_gc(p9, 3, descr=<ArrayP 8>)
+0# +172: p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_cells 40>)
+0# +172: label(p0, p1, p2, p3, p4, i5, p6, i7, i8, p11, p13, p15, p17, descr=TargetToken(140003404595232))
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# +251: guard_value(i7, 2, descr=<Guard0x7f551532e020>) [i7, p1, p0, p2, p3, p4, i5, p6, i8, p11, p13, p15, p17]
+0# +261: guard_nonnull_class(p11, ConstClass(W_IntObject), descr=<Guard0x7f551538c638>) [p1, p0, p11, p2, p3, p4, i5, p6, p13, p15, p17]
+0# +279: guard_value(i5, 0, descr=<Guard0x7f551538c5c0>) [i5, p1, p0, p2, p3, p4, p6, p11, p13, p17]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #12 LOAD_CONST')
+0# +289: guard_value(p4, ConstPtr(ptr22), descr=<Guard0x7f551538c548>) [p1, p0, p4, p2, p3, p6, p11, p13, p17]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #15 COMPARE_OP')
+0# +308: i23 = getfield_gc_pure(p11, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval 8>)
+0# +312: i25 = int_lt(i23, 10)
+0# guard_true(i25, descr=<Guard0x7f551538c4d0>) [p1, p0, p11, p2, p3, p6, p13]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #18 POP_JUMP_IF_FALSE')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #21 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #24 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #27 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #30 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #33 INPLACE_ADD')
+0# +322: i27 = int_add(i23, 1)
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #34 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #37 JUMP_ABSOLUTE')
+0# +327: guard_not_invalidated(descr=<Guard0x7f551538c458>) [p1, p0, p2, p3, p6, i27]
+0# +327: i29 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +335: i31 = int_lt(i29, 0)
+0# guard_false(i31, descr=<Guard0x7f551538c3e0>) [p1, p0, p2, p3, p6, i27]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# +345: label(p0, p1, p2, p3, p6, i27, descr=TargetToken(140003404595320))
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #12 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #15 COMPARE_OP')
+0# +376: i32 = int_lt(i27, 10)
+0# guard_true(i32, descr=<Guard0x7f551538c368>) [p1, p0, p2, p3, p6, i27]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #18 POP_JUMP_IF_FALSE')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #21 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #24 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #27 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #30 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #33 INPLACE_ADD')
+0# +386: i33 = int_add(i27, 1)
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #34 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #37 JUMP_ABSOLUTE')
+0# +390: guard_not_invalidated(descr=<Guard0x7f551538c2f0>) [p1, p0, p2, p3, p6, i33, None]
+0# +390: i35 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +398: i36 = int_lt(i35, 0)
+0# guard_false(i36, descr=<Guard0x7f551538c278>) [p1, p0, p2, p3, p6, i33, None]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# +408: jump(p0, p1, p2, p3, p6, i33, descr=TargetToken(140003404595320))
+0# +416: --end of the loop--
+0# [1cffd9ba83b9] jit-log-noopt-loop}
+0# [1cffd9d7af1e] {jit-backend
+0# [1cffd9ea4873] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32128 +0  4881EC9800000048896C24604889FD48895C24684C896424704C896C24784C89B424800000004C89BC2488000000488B0425B07916034829E0483B0425109F0103760D49BBD61BB314557F000041FFD349BBD8509317557F00004D8B3B4D8D770149BBD8509317557F00004D89334C8BB5380100004D8BBE800000004D8B6E504D8B66704D0FB6968E0000004D8B4E604D8B4678498B7E58498B7668488B5E10488B5618488B4620488B4E284C89BD480100004C89AD500100004C898D580100004889BD6001000048899D6801000048898D7001000049BB38519317557F0000498B0B488D590149BB38519317557F000049891B4983F8030F85000000008138E82200000F85000000004C8B40104D85C00F8400000000488B5808498B48108139685505000F8500000000498B48084C8B4108488B79104C8B49184883FB000F8C000000004C39CB0F8D000000004889D9480FAFDF4D89C54901D8488D5901488958084983FA000F85000000004C8B521041813A089203000F85000000004C8B5208498B4A084C8D79014C8985600100004C89A56801000048898D700100004C898D78010000488985800100004C8995880100004889BD90010000488995980100004C89D74C89FE49BB8820B314557F00004C895D2041BBE060730041FFD3F6450401740D49BBFD11B314557F000041FFD348C745200000000048833C2500E61503000F8500000000488B9588010000488B7A104C8B9560010000488B85700100004C8954C710488B0425807816034883F8000F8C00000000488B856801000049BB584C3815557F00004C39D80F850000000049BB50519317557F00004D8B13498D420149BB50519317557F0000498903483B9D780100000F8D000000004889D8480FAF9D900100004D89EA4901DD488D5801488B4208488D780148899560010000488985680100004C8995700100004889FE4889D749BBE820B314557F00004C895D2041BBE060730041FFD3F6450401740D49BBFD11B314557F000041FFD348C74520000000004C8B958001000049895A0848833C2500E61503000F8500000000488B8560010000488B5010488BBD680100004C896CFA10488B3C25807816034883FF000F8C000000004C89AD600100004C8995800100004C8BAD700100004889C2E90BFFFFFF49BBA0967F17557F0000415349BB0820B314557F0000415349BB0010B314557F000041FFE349BB704A8317557F0000415349BB1820B314557F0000415349BB0010B314557F000041FFE349BBF8498317557F0000415349BB2820B314557F0000415349BB0010B314557F000041FFE349BB80498317557F0000415349BB3820B314557F0000415349BB0010B314557F000041FFE349BB08498317557F0000415349BB4820B314557F0000415349BB0010B314557F000041FFE349BB90488317557F0000415349BB5820B314557F0000415349BB0010B314557F000041FFE349BB18488317557F0000415349BB6820B314557F0000415349BB0010B314557F000041FFE349BBA0478317557F0000415349BB7820B314557F0000415349BB0010B314557F000041FFE349BB28478317557F0000415349BB9820B314557F0000415349BB8510B314557F000041FFE349BBB0468317557F0000415349BBA820B314557F0000415349BB0010B314557F000041FFE349BB38468317557F0000415349BBB820B314557F0000415349BB0010B314557F000041FFE349BBC0458317557F0000415349BBC820B314557F0000415349BB0010B314557F000041FFE349BB48458317557F0000415349BBD820B314557F0000415349BB0010B314557F000041FFE349BBD0448317557F0000415349BBF820B314557F0000415349BB8510B314557F000041FFE349BB58448317557F0000415349BB0821B314557F0000415349BB0010B314557F000041FFE349BBE0438317557F0000415349BB1821B314557F0000415349BB0010B314557F000041FFE3
+0# [1cffd9ebc29f] jit-backend-dump}
+0# [1cffd9ebcab0] {jit-backend-addr
+0# Loop 1 (<code object h, file 'x.py', line 14> #13 FOR_ITER) has address 0x7f5514b32178 to 0x7f5514b32470 (bootstrap 0x7f5514b32128)
+0# [1cffd9ebde77] jit-backend-addr}
+0# [1cffd9ebe969] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32222 +0  4A020000
+0# [1cffd9ebfa23] jit-backend-dump}
+0# [1cffd9ec0059] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3222e +0  63020000
+0# [1cffd9ec0ae6] jit-backend-dump}
+0# [1cffd9ec0f4f] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3223b +0  7B020000
+0# [1cffd9ec18bc] jit-backend-dump}
+0# [1cffd9ec1d28] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3224f +0  8C020000
+0# [1cffd9ec2689] jit-backend-dump}
+0# [1cffd9ec2b07] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32269 +0  97020000
+0# [1cffd9ec3474] jit-backend-dump}
+0# [1cffd9ec38bc] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32272 +0  B3020000
+0# [1cffd9ec4220] jit-backend-dump}
+0# [1cffd9ec4677] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32291 +0  B9020000
+0# [1cffd9ec5011] jit-backend-dump}
+0# [1cffd9ec5459] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b322a2 +0  CD020000
+0# [1cffd9ec5e1c] jit-backend-dump}
+0# [1cffd9ec6279] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3232d +0  67020000
+0# [1cffd9ec6bd7] jit-backend-dump}
+0# [1cffd9ec77c9] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3235d +0  81020000
+0# [1cffd9ec8142] jit-backend-dump}
+0# [1cffd9ec85c0] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32377 +0  8C020000
+0# [1cffd9ecbf8d] jit-backend-dump}
+0# [1cffd9ecc51d] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b323a2 +0  86020000
+0# [1cffd9eccee0] jit-backend-dump}
+0# [1cffd9ecd33a] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32426 +0  27020000
+0# [1cffd9ecdc8f] jit-backend-dump}
+0# [1cffd9ece160] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3244f +0  48020000
+0# [1cffd9eceab8] jit-backend-dump}
+0# [1cffd9ecf545] jit-backend}
+0# [1cffd9ed0c35] {jit-log-noopt-loop
+0# # Loop 1 (<code object h, file 'x.py', line 14> #13 FOR_ITER) : loop with 82 ops
+0# [p0, p1]
+0# +110: p2 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_w_f_trace 128>)
+0# +124: p3 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_last_exception 80>)
+0# +128: p4 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_pycode 112>)
+0# +132: i5 = getfield_gc(p0, descr=<FieldU pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 142>)
+0# +140: p6 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_lastblock 96>)
+0# +144: i7 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_valuestackdepth 120>)
+0# +148: i8 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_last_instr 88>)
+0# +152: p9 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_locals_stack_w 104>)
+0# +156: p11 = getarrayitem_gc(p9, 0, descr=<ArrayP 8>)
+0# +160: p13 = getarrayitem_gc(p9, 1, descr=<ArrayP 8>)
+0# +164: p15 = getarrayitem_gc(p9, 2, descr=<ArrayP 8>)
+0# +168: p17 = getarrayitem_gc(p9, 3, descr=<ArrayP 8>)
+0# +172: p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_cells 40>)
+0# +172: label(p0, p1, p2, p3, p4, i5, p6, i7, i8, p11, p13, p15, p17, descr=TargetToken(140003443320224))
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +244: guard_value(i7, 3, descr=<Guard0x7f55177f96a0>) [i7, p1, p0, p2, p3, p4, i5, p6, i8, p11, p13, p15, p17]
+0# +254: guard_class(p15, 26177128, descr=<Guard0x7f5517834a70>) [p1, p0, p15, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +266: p21 = getfield_gc(p15, descr=<FieldP pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_w_seq 16>)
+0# +270: guard_nonnull(p21, descr=<Guard0x7f55178349f8>) [p1, p0, p15, p21, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +279: i22 = getfield_gc(p15, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
+0# +283: p23 = getfield_gc(p21, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_strategy 16>)
+0# +287: guard_class(p23, 26517736, descr=<Guard0x7f5517834980>) [p1, p0, p15, i22, p23, p21, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +299: p25 = getfield_gc(p21, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_lstorage 8>)
+0# +303: i26 = getfield_gc_pure(p25, descr=<FieldS tuple3.item0 8>)
+0# +307: i27 = getfield_gc_pure(p25, descr=<FieldS tuple3.item1 16>)
+0# +311: i28 = getfield_gc_pure(p25, descr=<FieldS tuple3.item2 24>)
+0# +315: i30 = int_lt(i22, 0)
+0# guard_false(i30, descr=<Guard0x7f5517834908>) [p1, p0, p15, i22, i28, i27, i26, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +325: i31 = int_ge(i22, i28)
+0# guard_false(i31, descr=<Guard0x7f5517834890>) [p1, p0, p15, i22, i27, i26, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +334: i32 = int_mul(i22, i27)
+0# +341: i33 = int_add(i26, i32)
+0# +347: i35 = int_add(i22, 1)
+0# +351: setfield_gc(p15, i35, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
+0# +355: guard_value(i5, 0, descr=<Guard0x7f5517834818>) [i5, p1, p0, p2, p3, p4, p6, p11, p13, p15, i33]
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #16 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #19 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #22 LIST_APPEND')
+0# +365: p37 = getfield_gc(p13, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_strategy 16>)
+0# +369: guard_class(p37, 26402184, descr=<Guard0x7f55178347a0>) [p1, p0, p37, p13, p2, p3, p4, p6, p15, i33]
+0# +382: p39 = getfield_gc(p13, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_lstorage 8>)
+0# +386: i40 = getfield_gc(p39, descr=<FieldS list.length 8>)
+0# +390: i42 = int_add(i40, 1)
+0# +394: p43 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +394: i44 = arraylen_gc(p43, descr=<ArrayS 8>)
+0# +394: call(ConstClass(_ll_list_resize_ge_trampoline__v672___simple_call__function__), p39, i42, descr=<Callv 0 ri EF=4>)
+0# +506: guard_no_exception(descr=<Guard0x7f5517834728>) [p1, p0, i40, i33, p39, p2, p3, p4, p6, p13, p15, None]
+0# +521: p47 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +532: setarrayitem_gc(p47, i40, i33, descr=<ArrayS 8>)
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #25 JUMP_ABSOLUTE')
+0# +551: guard_not_invalidated(descr=<Guard0x7f55178346b0>) [p1, p0, p2, p3, p4, p6, p13, p15, i33]
+0# +551: i49 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +559: i51 = int_lt(i49, 0)
+0# guard_false(i51, descr=<Guard0x7f5517834638>) [p1, p0, p2, p3, p4, p6, p13, p15, i33]
+0# +569: guard_value(p4, ConstPtr(ptr52), descr=<Guard0x7f55178345c0>) [p1, p0, p4, p2, p3, p6, p13, p15, i33]
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +595: label(p0, p1, p2, p3, p6, i33, p13, p15, i35, i28, i27, i26, p39, descr=TargetToken(140003443320312))
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +625: i53 = int_ge(i35, i28)
+0# guard_false(i53, descr=<Guard0x7f5517834548>) [p1, p0, p15, i35, i27, i26, p2, p3, p6, p13, i33]
+0# +638: i54 = int_mul(i35, i27)
+0# +649: i55 = int_add(i26, i54)
+0# +655: i56 = int_add(i35, 1)
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #16 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #19 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #22 LIST_APPEND')
+0# +659: i57 = getfield_gc(p39, descr=<FieldS list.length 8>)
+0# +663: i58 = int_add(i57, 1)
+0# +667: p59 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +667: i60 = arraylen_gc(p59, descr=<ArrayS 8>)
+0# +667: call(ConstClass(_ll_list_resize_ge_trampoline__v672___simple_call__function__), p39, i58, descr=<Callv 0 ri EF=4>)
+0# +744: setfield_gc(p15, i56, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
+0# +755: guard_no_exception(descr=<Guard0x7f55178344d0>) [p1, p0, i57, i55, p39, p2, p3, p6, p13, p15, None]
+0# +770: p61 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +781: setarrayitem_gc(p61, i57, i55, descr=<ArrayS 8>)
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #25 JUMP_ABSOLUTE')
+0# +793: guard_not_invalidated(descr=<Guard0x7f5517834458>) [p1, p0, p2, p3, p6, p13, p15, i55, None]
+0# +793: i62 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +801: i63 = int_lt(i62, 0)
+0# guard_false(i63, descr=<Guard0x7f55178343e0>) [p1, p0, p2, p3, p6, p13, p15, i55, None]
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +811: jump(p0, p1, p2, p3, p6, i55, p13, p15, i56, i28, i27, i26, p39, descr=TargetToken(140003443320312))
+0# +840: --end of the loop--
+0# [1cffd9f27224] jit-log-noopt-loop}
+0# [1cffd9f6f244] {jit-backend-counts
+0# entry 0:1
+0# TargetToken(140003404595232):1
+0# TargetToken(140003404595320):4
+0# entry 1:1
+0# TargetToken(140003443320224):1
+0# TargetToken(140003443320312):4
+0# [1cffd9f72430] jit-backend-counts}
\ No newline at end of file
diff --git a/pypy/tool/jitlogparser/test/logtest_rewritten.log b/pypy/tool/jitlogparser/test/logtest_rewritten.log
new file mode 100644
--- /dev/null
+++ b/pypy/tool/jitlogparser/test/logtest_rewritten.log
@@ -0,0 +1,356 @@
+0# [1cffd8feb691] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31000 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd8ffaba6] jit-backend-dump}
+0# [1cffd90012ee] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31085 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000488B1C2508E6150348C7042500E615030000000048C7042508E615030000000048895D38584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd9003b76] jit-backend-dump}
+0# [1cffd900719f] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3112e +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000488B7C240841BBB064120141FFD3488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000C20800
+0# [1cffd9008c81] jit-backend-dump}
+0# [1cffd900b384] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31191 +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000488B7C240841BBF065120141FFD3488B442408F6400480488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000C20800
+0# [1cffd900cf18] jit-backend-dump}
+0# [1cffd9010345] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b311fd +0  4883EC384889442408F20F114424184889EF48895C24284C89642430488B1C2508E615034C8B242500E6150348C7042500E615030000000048C7042508E615030000000041BBB064120141FFD3F20F10442418488B44240848891C2508E615034C89242500E61503488B5C24284C8B642430488D642438C3
+0# [1cffd9011f0b] jit-backend-dump}
+0# [1cffd9015bd8] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31275 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B4C240848894D20488B7424104889EF4883EC0848C7452810000000488B0C2508E6150348894D38488B1C2500E6150348C7042500E615030000000048C7042508E615030000000041BB60DBE80041FFD34889C5488B4D3848C745380000000048890C2508E6150348891C2500E615034883C40848C745280000000048C7452000000000488B4D58488B4560488B5568488B5D70488B7578488BBD800000004C8B85880000004C8B8D900000004C8B95980000004C8BA5A00000004C8BADA80000004C8BB5B00000004C8BBDB8000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C3
+0# [1cffd901a191] jit-backend-dump}
+0# [1cffd901b3a6] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31491 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd901dc46] jit-backend-dump}
+0# [1cffd901ef79] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31595 +0  48894D58488945604889556848895D70488975784889BD800000004C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B1C2508E6150348C7042500E615030000000048C7042508E615030000000048895D38584889452058488945104889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd902ce01] jit-backend-dump}
+0# [1cffd902e819] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b316bd +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B7C240841BBB064120141FFD3488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C20800
+0# [1cffd9031b79] jit-backend-dump}
+0# [1cffd90331b0] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3181e +0  4889455848894D60488955684889757048897D784C8985800000004C898D880000004C899590000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B7C240841BBF065120141FFD3488B442408F6400480488B4558488B4D60488B5568488B7570488B7D784C8B85800000004C8B8D880000004C8B9590000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000C20800
+0# [1cffd903629a] jit-backend-dump}
+0# [1cffd903736b] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31988 +0  488B042508E6150348C7042500E615030000000048C7042508E61503000000004889453848C7451000C2B5014889E84C8BBC24880000004C8BB424800000004C8B6C24784C8B642470488B5C2468488B6C24604881C498000000C3
+0# [1cffd9038a70] jit-backend-dump}
+0# [1cffd903e2cd] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b319e3 +0  48894D584889556848895D70488975784C8985880000004C898D900000004C8995980000004C89A5A00000004C89ADA80000004C89B5B00000004C89BDB8000000F20F1185C0000000F20F118DC8000000F20F1195D0000000F20F119DD8000000F20F11A5E0000000F20F11ADE8000000F20F11B5F0000000F20F11BDF8000000F2440F118500010000F2440F118D08010000F2440F119510010000F2440F119D18010000F2440F11A520010000F2440F11AD28010000F2440F11B530010000488B4C240848894D204829C74883EC0848C745281000000041BBB0A4E80041FFD34883C4084885C00F84F4000000F645040174154883EC0849BBFD11B314557F000041FFD34883C40848C7452800000000488B4D58488B5568488B5D70488B75784C8B85880000004C8B8D900000004C8B95980000004C8BA5A00000004C8BADA80000004C8BB5B00000004C8BBDB8000000F20F1085C0000000F20F108DC8000000F20F1095D0000000F20F109DD8000000F20F10A5E0000000F20F10ADE8000000F20F10B5F0000000F20F10BDF8000000F2440F108500010000F2440F108D08010000F2440F109510010000F2440F109D18010000F2440F10A520010000F2440F10AD28010000F2440F10B530010000488B3C25484CB60148C7452000000000C34883C40849BB8819B314557F000041FFE3
+0# [1cffd904265b] jit-backend-dump}
+0# [1cffd90448f2] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31bd6 +0  4889E74883EC0841BBD00F210141FFD34883C408488B042500E615034885C07501C34883C40849BB8819B314557F000041FFE3
+0# [1cffd9045d15] jit-backend-dump}
+0# [1cffd904647a] {jit-backend-counts
+0# [1cffd9046851] jit-backend-counts}
+0# [1cffd9636773] {jit-backend
+0# [1cffd9afbdde] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31ce0 +0  4881EC9800000048896C24604889FD48895C24684C896424704C896C24784C89B424800000004C89BC2488000000488B0425B07916034829E0483B0425109F0103760D49BBD61BB314557F000041FFD349BBF0509317557F00004D8B3B4D8D770149BBF0509317557F00004D89334C8BB5380100004D8BBE800000004D8B6E504D8B66704D0FB6968E0000004D8B4E604D8B4678498B7E58498B7668488B5E10488B5618488B4620488B4E284C89BD480100004C89AD500100004C898D580100004889BD60010000488995680100004889857001000048898D7801000049BB08519317557F0000498B0B488D410149BB08519317557F00004989034983F8020F85000000004883FB017206813B180C00000F85000000004983FA000F850000000049BB20803215557F00004D39DC0F85000000004C8B63084983FC0A0F8D00000000498D5C24014C8B2425807816034983FC000F8C0000000049BB20519317557F00004D8B234D8D54240149BB20519317557F00004D89134883FB0A0F8D000000004C8D5301488B1C25807816034883FB000F8C000000004C89D3E9B9FFFFFF49BB20E03215557F0000415349BB401CB314557F0000415349BB0010B314557F000041FFE349BB38C63815557F0000415349BB501CB314557F0000415349BB0010B314557F000041FFE349BBC0C53815557F0000415349BB601CB314557F0000415349BB0010B314557F000041FFE349BB48C53815557F0000415349BB701CB314557F0000415349BB0010B314557F000041FFE349BBD0C43815557F0000415349BB801CB314557F0000415349BB0010B314557F000041FFE349BB58C43815557F0000415349BB901CB314557F0000415349BB0010B314557F000041FFE349BBE0C33815557F0000415349BBA01CB314557F0000415349BB0010B314557F000041FFE349BB68C33815557F0000415349BBB01CB314557F0000415349BB0010B314557F000041FFE349BBF0C23815557F0000415349BBC01CB314557F0000415349BB0010B314557F000041FFE349BB78C23815557F0000415349BBD01CB314557F0000415349BB0010B314557F000041FFE3
+0# [1cffd9b146d6] jit-backend-dump}
+0# [1cffd9b14ff3] {jit-backend-addr
+0# Loop 0 (<code object g, file 'x.py', line 8> #9 LOAD_FAST) has address 0x7f5514b31d30 to 0x7f5514b31e80 (bootstrap 0x7f5514b31ce0)
+0# [1cffd9b16753] jit-backend-addr}
+0# [1cffd9b17245] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31de1 +0  9B000000
+0# [1cffd9b18103] jit-backend-dump}
+0# [1cffd9b18762] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31df3 +0  AE000000
+0# [1cffd9b191ae] jit-backend-dump}
+0# [1cffd9b1960b] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31dfd +0  C9000000
+0# [1cffd9b19f1f] jit-backend-dump}
+0# [1cffd9b1a32f] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e10 +0  DB000000
+0# [1cffd9b1ac8d] jit-backend-dump}
+0# [1cffd9b1b091] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e1e +0  F2000000
+0# [1cffd9b1ba54] jit-backend-dump}
+0# [1cffd9b1bfec] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e35 +0  25010000
+0# [1cffd9b1c8d3] jit-backend-dump}
+0# [1cffd9b1ccfb] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e5e +0  21010000
+0# [1cffd9b1d5d6] jit-backend-dump}
+0# [1cffd9b1da25] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b31e74 +0  55010000
+0# [1cffd9b1e3b8] jit-backend-dump}
+0# [1cffd9b1f0b2] jit-backend}
+0# [1cffd9b20d9f] {jit-log-rewritten-loop
+0# # Loop 0 (<code object g, file 'x.py', line 8> #9 LOAD_FAST) : loop with 59 ops
+0# [p0, p1]
+0# +110: p2 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_w_f_trace 128>)
+0# +124: p3 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_last_exception 80>)
+0# +128: p4 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_pycode 112>)
+0# +132: i5 = getfield_gc(p0, descr=<FieldU pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 142>)
+0# +140: p6 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_lastblock 96>)
+0# +144: i7 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_valuestackdepth 120>)
+0# +148: i8 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_last_instr 88>)
+0# +152: p9 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_locals_stack_w 104>)
+0# +156: p11 = getarrayitem_gc(p9, 0, descr=<ArrayP 8>)
+0# +160: p13 = getarrayitem_gc(p9, 1, descr=<ArrayP 8>)
+0# +164: p15 = getarrayitem_gc(p9, 2, descr=<ArrayP 8>)
+0# +168: p17 = getarrayitem_gc(p9, 3, descr=<ArrayP 8>)
+0# +172: p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_cells 40>)
+0# +172: label(p0, p1, p2, p3, p4, i5, p6, i7, i8, p11, p13, p15, p17, descr=TargetToken(140003404595232))
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# +251: guard_value(i7, 2, descr=<Guard0x7f551532e020>) [i7, p1, p0, p2, p3, p4, i5, p6, i8, p11, p13, p15, p17]
+0# +261: guard_nonnull_class(p11, ConstClass(W_IntObject), descr=<Guard0x7f551538c638>) [p1, p0, p11, p2, p3, p4, i5, p6, p13, p15, p17]
+0# +279: guard_value(i5, 0, descr=<Guard0x7f551538c5c0>) [i5, p1, p0, p2, p3, p4, p6, p11, p13, p17]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #12 LOAD_CONST')
+0# +289: guard_value(p4, ConstPtr(ptr22), descr=<Guard0x7f551538c548>) [p1, p0, p4, p2, p3, p6, p11, p13, p17]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #15 COMPARE_OP')
+0# +308: i23 = getfield_gc_pure(p11, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval 8>)
+0# +312: i25 = int_lt(i23, 10)
+0# guard_true(i25, descr=<Guard0x7f551538c4d0>) [p1, p0, p11, p2, p3, p6, p13]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #18 POP_JUMP_IF_FALSE')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #21 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #24 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #27 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #30 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #33 INPLACE_ADD')
+0# +322: i27 = int_add(i23, 1)
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #34 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #37 JUMP_ABSOLUTE')
+0# +327: guard_not_invalidated(descr=<Guard0x7f551538c458>) [p1, p0, p2, p3, p6, i27]
+0# +327: i29 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +335: i31 = int_lt(i29, 0)
+0# guard_false(i31, descr=<Guard0x7f551538c3e0>) [p1, p0, p2, p3, p6, i27]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# +345: label(p0, p1, p2, p3, p6, i27, descr=TargetToken(140003404595320))
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #12 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #15 COMPARE_OP')
+0# +376: i32 = int_lt(i27, 10)
+0# guard_true(i32, descr=<Guard0x7f551538c368>) [p1, p0, p2, p3, p6, i27]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #18 POP_JUMP_IF_FALSE')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #21 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #24 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #27 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #30 LOAD_CONST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #33 INPLACE_ADD')
+0# +386: i33 = int_add(i27, 1)
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #34 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #37 JUMP_ABSOLUTE')
+0# +390: guard_not_invalidated(descr=<Guard0x7f551538c2f0>) [p1, p0, p2, p3, p6, i33, None]
+0# +390: i35 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +398: i36 = int_lt(i35, 0)
+0# guard_false(i36, descr=<Guard0x7f551538c278>) [p1, p0, p2, p3, p6, i33, None]
+0# debug_merge_point(0, 0, '<code object g. file 'x.py'. line 8> #9 LOAD_FAST')
+0# +408: jump(p0, p1, p2, p3, p6, i33, descr=TargetToken(140003404595320))
+0# +416: --end of the loop--
+0# [1cffd9ba83b9] jit-log-rewritten-loop}
+0# [1cffd9d7af1e] {jit-backend
+0# [1cffd9ea4873] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32128 +0  4881EC9800000048896C24604889FD48895C24684C896424704C896C24784C89B424800000004C89BC2488000000488B0425B07916034829E0483B0425109F0103760D49BBD61BB314557F000041FFD349BBD8509317557F00004D8B3B4D8D770149BBD8509317557F00004D89334C8BB5380100004D8BBE800000004D8B6E504D8B66704D0FB6968E0000004D8B4E604D8B4678498B7E58498B7668488B5E10488B5618488B4620488B4E284C89BD480100004C89AD500100004C898D580100004889BD6001000048899D6801000048898D7001000049BB38519317557F0000498B0B488D590149BB38519317557F000049891B4983F8030F85000000008138E82200000F85000000004C8B40104D85C00F8400000000488B5808498B48108139685505000F8500000000498B48084C8B4108488B79104C8B49184883FB000F8C000000004C39CB0F8D000000004889D9480FAFDF4D89C54901D8488D5901488958084983FA000F85000000004C8B521041813A089203000F85000000004C8B5208498B4A084C8D79014C8985600100004C89A56801000048898D700100004C898D78010000488985800100004C8995880100004889BD90010000488995980100004C89D74C89FE49BB8820B314557F00004C895D2041BBE060730041FFD3F6450401740D49BBFD11B314557F000041FFD348C745200000000048833C2500E61503000F8500000000488B9588010000488B7A104C8B9560010000488B85700100004C8954C710488B0425807816034883F8000F8C00000000488B856801000049BB584C3815557F00004C39D80F850000000049BB50519317557F00004D8B13498D420149BB50519317557F0000498903483B9D780100000F8D000000004889D8480FAF9D900100004D89EA4901DD488D5801488B4208488D780148899560010000488985680100004C8995700100004889FE4889D749BBE820B314557F00004C895D2041BBE060730041FFD3F6450401740D49BBFD11B314557F000041FFD348C74520000000004C8B958001000049895A0848833C2500E61503000F8500000000488B8560010000488B5010488BBD680100004C896CFA10488B3C25807816034883FF000F8C000000004C89AD600100004C8995800100004C8BAD700100004889C2E90BFFFFFF49BBA0967F17557F0000415349BB0820B314557F0000415349BB0010B314557F000041FFE349BB704A8317557F0000415349BB1820B314557F0000415349BB0010B314557F000041FFE349BBF8498317557F0000415349BB2820B314557F0000415349BB0010B314557F000041FFE349BB80498317557F0000415349BB3820B314557F0000415349BB0010B314557F000041FFE349BB08498317557F0000415349BB4820B314557F0000415349BB0010B314557F000041FFE349BB90488317557F0000415349BB5820B314557F0000415349BB0010B314557F000041FFE349BB18488317557F0000415349BB6820B314557F0000415349BB0010B314557F000041FFE349BBA0478317557F0000415349BB7820B314557F0000415349BB0010B314557F000041FFE349BB28478317557F0000415349BB9820B314557F0000415349BB8510B314557F000041FFE349BBB0468317557F0000415349BBA820B314557F0000415349BB0010B314557F000041FFE349BB38468317557F0000415349BBB820B314557F0000415349BB0010B314557F000041FFE349BBC0458317557F0000415349BBC820B314557F0000415349BB0010B314557F000041FFE349BB48458317557F0000415349BBD820B314557F0000415349BB0010B314557F000041FFE349BBD0448317557F0000415349BBF820B314557F0000415349BB8510B314557F000041FFE349BB58448317557F0000415349BB0821B314557F0000415349BB0010B314557F000041FFE349BBE0438317557F0000415349BB1821B314557F0000415349BB0010B314557F000041FFE3
+0# [1cffd9ebc29f] jit-backend-dump}
+0# [1cffd9ebcab0] {jit-backend-addr
+0# Loop 1 (<code object h, file 'x.py', line 14> #13 FOR_ITER) has address 0x7f5514b32178 to 0x7f5514b32470 (bootstrap 0x7f5514b32128)
+0# [1cffd9ebde77] jit-backend-addr}
+0# [1cffd9ebe969] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32222 +0  4A020000
+0# [1cffd9ebfa23] jit-backend-dump}
+0# [1cffd9ec0059] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3222e +0  63020000
+0# [1cffd9ec0ae6] jit-backend-dump}
+0# [1cffd9ec0f4f] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3223b +0  7B020000
+0# [1cffd9ec18bc] jit-backend-dump}
+0# [1cffd9ec1d28] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3224f +0  8C020000
+0# [1cffd9ec2689] jit-backend-dump}
+0# [1cffd9ec2b07] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32269 +0  97020000
+0# [1cffd9ec3474] jit-backend-dump}
+0# [1cffd9ec38bc] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32272 +0  B3020000
+0# [1cffd9ec4220] jit-backend-dump}
+0# [1cffd9ec4677] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32291 +0  B9020000
+0# [1cffd9ec5011] jit-backend-dump}
+0# [1cffd9ec5459] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b322a2 +0  CD020000
+0# [1cffd9ec5e1c] jit-backend-dump}
+0# [1cffd9ec6279] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3232d +0  67020000
+0# [1cffd9ec6bd7] jit-backend-dump}
+0# [1cffd9ec77c9] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3235d +0  81020000
+0# [1cffd9ec8142] jit-backend-dump}
+0# [1cffd9ec85c0] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32377 +0  8C020000
+0# [1cffd9ecbf8d] jit-backend-dump}
+0# [1cffd9ecc51d] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b323a2 +0  86020000
+0# [1cffd9eccee0] jit-backend-dump}
+0# [1cffd9ecd33a] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b32426 +0  27020000
+0# [1cffd9ecdc8f] jit-backend-dump}
+0# [1cffd9ece160] {jit-backend-dump
+0# BACKEND x86_64
+0# SYS_EXECUTABLE python
+0# CODE_DUMP @7f5514b3244f +0  48020000
+0# [1cffd9eceab8] jit-backend-dump}
+0# [1cffd9ecf545] jit-backend}
+0# [1cffd9ed0c35] {jit-log-rewritten-loop
+0# # Loop 1 (<code object h, file 'x.py', line 14> #13 FOR_ITER) : loop with 82 ops
+0# [p0, p1]
+0# +110: p2 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_w_f_trace 128>)
+0# +124: p3 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_last_exception 80>)
+0# +128: p4 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_pycode 112>)
+0# +132: i5 = getfield_gc(p0, descr=<FieldU pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 142>)
+0# +140: p6 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_lastblock 96>)
+0# +144: i7 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_valuestackdepth 120>)
+0# +148: i8 = getfield_gc(p0, descr=<FieldS pypy.interpreter.pyframe.PyFrame.inst_last_instr 88>)
+0# +152: p9 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_locals_stack_w 104>)
+0# +156: p11 = getarrayitem_gc(p9, 0, descr=<ArrayP 8>)
+0# +160: p13 = getarrayitem_gc(p9, 1, descr=<ArrayP 8>)
+0# +164: p15 = getarrayitem_gc(p9, 2, descr=<ArrayP 8>)
+0# +168: p17 = getarrayitem_gc(p9, 3, descr=<ArrayP 8>)
+0# +172: p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_cells 40>)
+0# +172: label(p0, p1, p2, p3, p4, i5, p6, i7, i8, p11, p13, p15, p17, descr=TargetToken(140003443320224))
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +244: guard_value(i7, 3, descr=<Guard0x7f55177f96a0>) [i7, p1, p0, p2, p3, p4, i5, p6, i8, p11, p13, p15, p17]
+0# +254: guard_class(p15, 26177128, descr=<Guard0x7f5517834a70>) [p1, p0, p15, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +266: p21 = getfield_gc(p15, descr=<FieldP pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_w_seq 16>)
+0# +270: guard_nonnull(p21, descr=<Guard0x7f55178349f8>) [p1, p0, p15, p21, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +279: i22 = getfield_gc(p15, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
+0# +283: p23 = getfield_gc(p21, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_strategy 16>)
+0# +287: guard_class(p23, 26517736, descr=<Guard0x7f5517834980>) [p1, p0, p15, i22, p23, p21, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +299: p25 = getfield_gc(p21, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_lstorage 8>)
+0# +303: i26 = getfield_gc_pure(p25, descr=<FieldS tuple3.item0 8>)
+0# +307: i27 = getfield_gc_pure(p25, descr=<FieldS tuple3.item1 16>)
+0# +311: i28 = getfield_gc_pure(p25, descr=<FieldS tuple3.item2 24>)
+0# +315: i30 = int_lt(i22, 0)
+0# guard_false(i30, descr=<Guard0x7f5517834908>) [p1, p0, p15, i22, i28, i27, i26, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +325: i31 = int_ge(i22, i28)
+0# guard_false(i31, descr=<Guard0x7f5517834890>) [p1, p0, p15, i22, i27, i26, p2, p3, p4, i5, p6, p11, p13, p17]
+0# +334: i32 = int_mul(i22, i27)
+0# +341: i33 = int_add(i26, i32)
+0# +347: i35 = int_add(i22, 1)
+0# +351: setfield_gc(p15, i35, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
+0# +355: guard_value(i5, 0, descr=<Guard0x7f5517834818>) [i5, p1, p0, p2, p3, p4, p6, p11, p13, p15, i33]
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #16 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #19 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #22 LIST_APPEND')
+0# +365: p37 = getfield_gc(p13, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_strategy 16>)
+0# +369: guard_class(p37, 26402184, descr=<Guard0x7f55178347a0>) [p1, p0, p37, p13, p2, p3, p4, p6, p15, i33]
+0# +382: p39 = getfield_gc(p13, descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_lstorage 8>)
+0# +386: i40 = getfield_gc(p39, descr=<FieldS list.length 8>)
+0# +390: i42 = int_add(i40, 1)
+0# +394: p43 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +394: i44 = arraylen_gc(p43, descr=<ArrayS 8>)
+0# +394: call(ConstClass(_ll_list_resize_ge_trampoline__v672___simple_call__function__), p39, i42, descr=<Callv 0 ri EF=4>)
+0# +506: guard_no_exception(descr=<Guard0x7f5517834728>) [p1, p0, i40, i33, p39, p2, p3, p4, p6, p13, p15, None]
+0# +521: p47 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +532: setarrayitem_gc(p47, i40, i33, descr=<ArrayS 8>)
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #25 JUMP_ABSOLUTE')
+0# +551: guard_not_invalidated(descr=<Guard0x7f55178346b0>) [p1, p0, p2, p3, p4, p6, p13, p15, i33]
+0# +551: i49 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +559: i51 = int_lt(i49, 0)
+0# guard_false(i51, descr=<Guard0x7f5517834638>) [p1, p0, p2, p3, p4, p6, p13, p15, i33]
+0# +569: guard_value(p4, ConstPtr(ptr52), descr=<Guard0x7f55178345c0>) [p1, p0, p4, p2, p3, p6, p13, p15, i33]
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +595: label(p0, p1, p2, p3, p6, i33, p13, p15, i35, i28, i27, i26, p39, descr=TargetToken(140003443320312))
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +625: i53 = int_ge(i35, i28)
+0# guard_false(i53, descr=<Guard0x7f5517834548>) [p1, p0, p15, i35, i27, i26, p2, p3, p6, p13, i33]
+0# +638: i54 = int_mul(i35, i27)
+0# +649: i55 = int_add(i26, i54)
+0# +655: i56 = int_add(i35, 1)
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #16 STORE_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #19 LOAD_FAST')
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #22 LIST_APPEND')
+0# +659: i57 = getfield_gc(p39, descr=<FieldS list.length 8>)
+0# +663: i58 = int_add(i57, 1)
+0# +667: p59 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +667: i60 = arraylen_gc(p59, descr=<ArrayS 8>)
+0# +667: call(ConstClass(_ll_list_resize_ge_trampoline__v672___simple_call__function__), p39, i58, descr=<Callv 0 ri EF=4>)
+0# +744: setfield_gc(p15, i56, descr=<FieldS pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_index 8>)
+0# +755: guard_no_exception(descr=<Guard0x7f55178344d0>) [p1, p0, i57, i55, p39, p2, p3, p6, p13, p15, None]
+0# +770: p61 = getfield_gc(p39, descr=<FieldP list.items 16>)
+0# +781: setarrayitem_gc(p61, i57, i55, descr=<ArrayS 8>)
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #25 JUMP_ABSOLUTE')
+0# +793: guard_not_invalidated(descr=<Guard0x7f5517834458>) [p1, p0, p2, p3, p6, p13, p15, i55, None]
+0# +793: i62 = getfield_raw(51804288, descr=<FieldS pypysig_long_struct.c_value 0>)
+0# +801: i63 = int_lt(i62, 0)
+0# guard_false(i63, descr=<Guard0x7f55178343e0>) [p1, p0, p2, p3, p6, p13, p15, i55, None]
+0# debug_merge_point(0, 0, '<code object h. file 'x.py'. line 14> #13 FOR_ITER')
+0# +811: jump(p0, p1, p2, p3, p6, i55, p13, p15, i56, i28, i27, i26, p39, descr=TargetToken(140003443320312))
+0# +840: --end of the loop--
+0# [1cffd9f27224] jit-log-rewritten-loop}
+0# [1cffd9f6f244] {jit-backend-counts
+0# entry 0:1
+0# TargetToken(140003404595232):1
+0# TargetToken(140003404595320):4
+0# entry 1:1
+0# TargetToken(140003443320224):1
+0# TargetToken(140003443320312):4
+0# [1cffd9f72430] jit-backend-counts}
\ No newline at end of file
diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -289,6 +289,26 @@
         loop.force_asm()
     assert 'cmp' in loops[1].operations[2].asm
 
+def test_import_log_rewritten():
+    if not autodetect().startswith('x86'):
+        py.test.skip('x86 only test')
+    _, loops = import_log(str(py.path.local(__file__).join(
+        '..', 'logtest_rewritten.log')))
+    for loop in loops:
+        loop.force_asm()
+    assert 'cmp' in loops[1].operations[2].asm
+
+def test_import_log_noopt():
+    if not autodetect().startswith('x86'):
+        py.test.skip('x86 only test')
+    _, loops = import_log(str(py.path.local(__file__).join(
+        '..', 'logtest_noopt.log')))
+    for loop in loops:
+        loop.force_asm()
+    assert 'cmp' in loops[1].operations[2].asm
+
+    
+
 def test_Op_repr_is_pure():
     op = Op('foobar', ['a', 'b'], 'c', 'mydescr')
     myrepr = 'c = foobar(a, b, descr=mydescr)'
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -15,10 +15,14 @@
     def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None, name=''):
         if type is None:
             debug_start("jit-log-noopt-loop")
+            debug_print("# Loop", number, '(%s)' % name, ":", "noopt",
+                        "with", len(operations), "ops")
             logops = self._log_operations(inputargs, operations, ops_offset)
             debug_stop("jit-log-noopt-loop")
         elif type == "rewritten":
             debug_start("jit-log-rewritten-loop")
+            debug_print("# Loop", number, '(%s)' % name, ":", type,
+                        "with", len(operations), "ops")
             logops = self._log_operations(inputargs, operations, ops_offset)
             debug_stop("jit-log-rewritten-loop")
         elif number == -2:
@@ -37,6 +41,9 @@
                    descr=None, ops_offset=None):
         if extra == "noopt":
             debug_start("jit-log-noopt-bridge")
+            debug_print("# bridge out of Guard",
+                        "0x%x" % compute_unique_id(descr),
+                        "with", len(operations), "ops")
             logops = self._log_operations(inputargs, operations, ops_offset)
             debug_stop("jit-log-noopt-bridge")
         elif extra == "compiling":
@@ -45,6 +52,9 @@
             debug_stop("jit-log-compiling-bridge")
         elif extra == "rewritten":
             debug_start("jit-log-rewritten-bridge")
+            debug_print("# bridge out of Guard",
+                        "0x%x" % compute_unique_id(descr),
+                        "with", len(operations), "ops")
             logops = self._log_operations(inputargs, operations, ops_offset)
             debug_stop("jit-log-rewritten-bridge")
         else:

From noreply at buildbot.pypy.org  Fri Oct 11 18:28:10 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 11 Oct 2013 18:28:10 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: logging fixes: use the right loop
	number
Message-ID: <20131011162810.488AF1C3654@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67323:d3b179cf66a2
Date: 2013-10-11 18:10 +0200
http://bitbucket.org/pypy/pypy/changeset/d3b179cf66a2/

Log:	logging fixes: use the right loop number

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -623,7 +623,7 @@
 
         ops_offset = self.mc.ops_offset
         if logger is not None:
-            logger.log_loop(inputargs, operations, 0, "rewritten",
+            logger.log_loop(inputargs, operations, looptoken.number, "rewritten",
                             name=loopname, ops_offset=ops_offset)
         self.teardown()
 
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -654,7 +654,7 @@
             looptoken._x86_ops_offset = ops_offset
         looptoken._ll_function_addr = rawstart
         if logger:
-            logger.log_loop(inputargs, operations, 0, "rewritten",
+            logger.log_loop(inputargs, operations, looptoken.number, "rewritten",
                             name=loopname, ops_offset=ops_offset)
 
         self.fixup_target_tokens(rawstart)

From noreply at buildbot.pypy.org  Fri Oct 11 18:28:11 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 11 Oct 2013 18:28:11 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: seems to fix one bug - reload frame
 after stm_invalidate_jmp_buf
Message-ID: <20131011162811.6D1501C3654@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67324:8291074f521b
Date: 2013-10-11 18:26 +0200
http://bitbucket.org/pypy/pypy/changeset/8291074f521b/

Log:	seems to fix one bug - reload frame after stm_invalidate_jmp_buf

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -80,8 +80,6 @@
         else:
             self.gc_size_of_header = WORD # for tests
         self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn)
-        self._build_failure_recovery(False, withfloats=False)
-        self._build_failure_recovery(True, withfloats=False)
         if gc_ll_descr.stm:
             descrs = [gc_ll_descr.P2Rdescr, gc_ll_descr.P2Wdescr]
         else:
@@ -90,6 +88,9 @@
             self._build_b_slowpath(d, False)
             self._build_b_slowpath(d, True)
             self._build_b_slowpath(d, False, for_frame=True)
+        # building the barriers needs to happen before these:
+        self._build_failure_recovery(False, withfloats=False)
+        self._build_failure_recovery(True, withfloats=False)
         # only for stm:
         if gc_ll_descr.stm:
             self._build_ptr_eq_slowpath()
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -439,6 +439,7 @@
         if not withcards:
             func = descr.get_barrier_fn(self.cpu, 
                                         returns_modified_object=is_stm)
+            assert func is not None
         else:
             assert not is_stm
             if descr.jit_wb_cards_set == 0:
@@ -929,6 +930,8 @@
             self.mc.LEA_rs(edi.value, FRAME_FIXED_SIZE * WORD)
             fn = stmtlocal.stm_invalidate_jmp_buf_fn
             self.mc.CALL(imm(self.cpu.cast_ptr_to_int(fn)))
+            # there could have been a collection in invalidate_jmp_buf()
+            self._reload_frame_if_necessary(self.mc)
 
         # the return value is the jitframe
         self.mc.MOV_rr(eax.value, ebp.value)
@@ -2531,6 +2534,7 @@
             mc.SUB_ri(esp.value, 16 - WORD) # erase the return address
             # ||retadr|...||
         func = descr.get_b_slowpath(helper_num)
+        assert func != 0
         mc.CALL(imm(func))
         # get result:
         if is_frame:

From noreply at buildbot.pypy.org  Fri Oct 11 19:21:58 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 19:21:58 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: More fixes,
 trying to minimize the diff with minimark.py.
Message-ID: <20131011172158.B2F4B1C021C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67325:3fb6c1306302
Date: 2013-10-11 19:01 +0200
http://bitbucket.org/pypy/pypy/changeset/3fb6c1306302/

Log:	More fixes, trying to minimize the diff with minimark.py.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -227,7 +227,7 @@
         # value of 128 means that card pages are 512 bytes (1024 on 64-bits)
         # in regular arrays of pointers; more in arrays whose items are
         # larger.  A value of 0 disables card marking.
-        "card_page_indices": 0,   # XXX was 128,
+        "card_page_indices": 128,
 
         # Objects whose total size is at least 'large_object' bytes are
         # allocated out of the nursery immediately, as old objects.  The
@@ -269,9 +269,6 @@
         self.max_heap_size_already_raised = False
         self.max_delta = float(r_uint(-1))
         #
-        if card_page_indices != 0:
-            import py
-            py.test.skip("cards unsupported")
         self.card_page_indices = card_page_indices
         if self.card_page_indices > 0:
             self.card_page_shift = 0
@@ -642,16 +639,10 @@
 
     def collect(self, gen=1):
         """Do a minor (gen=0) or full major (gen>0) collection."""
-        self.minor_collection()
         if gen > 0:
-            #
-            # First, finish the current major gc, if there is one in progress.
-            # This is a no-op if the gc_state is already STATE_SCANNING.
-            self.gc_step_until(STATE_SCANNING)
-            #
-            # Then do a complete collection again.
-            self.gc_step_until(STATE_MARKING)
-            self.gc_step_until(STATE_SCANNING)
+            self.minor_and_major_collection()
+        else:
+            self.minor_collection()
 
     def move_nursery_top(self, totalsize):
         size = self.nursery_cleanup
@@ -1000,7 +991,6 @@
         """
         return self.ac.total_memory_used + self.rawmalloced_total_size
 
-
     def card_marking_words_for_length(self, length):
         # --- Unoptimized version:
         #num_bits = ((length-1) >> self.card_page_shift) + 1
@@ -1061,7 +1051,9 @@
             self.trace(obj, self._debug_check_not_white, None)
             # During marking, all visited (black) objects should always have
             # the GCFLAG_TRACK_YOUNG_PTRS flag set, for the write barrier to
-            # trigger --- at least if they contain any gc ptr
+            # trigger --- at least if they contain any gc ptr.  We are just
+            # after a minor or major collection here, so we can't see the
+            # object state VISITED & ~WRITE_BARRIER.
             typeid = self.get_type_id(obj)
             if self.has_gcptr(typeid):
                 ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS != 0,
@@ -1167,7 +1159,10 @@
 
     def write_barrier_from_array(self, addr_array, index):
         if self.header(addr_array).tid & GCFLAG_TRACK_YOUNG_PTRS:
-            self.remember_young_pointer(addr_array)
+            if self.card_page_indices > 0:
+                self.remember_young_pointer_from_array2(addr_array, index)
+            else:
+                self.remember_young_pointer(addr_array)
 
     def _init_writebarrier_logic(self):
         DEBUG = self.DEBUG
@@ -1286,19 +1281,6 @@
         return llarena.getfakearenaaddress(addr_byte) + (~byteindex)
 
 
-    def assume_young_pointers(self, addr_struct):
-        """Called occasionally by the JIT to mean ``assume that 'addr_struct'
-        may now contain young pointers.''
-        """
-        objhdr = self.header(addr_struct)
-        if objhdr.tid & GCFLAG_TRACK_YOUNG_PTRS:
-            self.old_objects_pointing_to_young.append(addr_struct)
-            objhdr.tid &= ~GCFLAG_TRACK_YOUNG_PTRS
-            #
-            if objhdr.tid & GCFLAG_NO_HEAP_PTRS:
-                objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS
-                self.prebuilt_root_objects.append(addr_struct)
-
     def writebarrier_before_copy(self, source_addr, dest_addr,
                                  source_start, dest_start, length):
         """ This has the same effect as calling writebarrier over
@@ -1313,8 +1295,6 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
-            if self.card_page_indices == 0:
-                return False     # shouldn't have GCFLAG_HAS_CARDS then...
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
@@ -1349,6 +1329,7 @@
 
     def manually_copy_card_bits(self, source_addr, dest_addr, length):
         # manually copy the individual card marks from source to dest
+        assert self.card_page_indices > 0
         bytes = self.card_marking_bytes_for_length(length)
         #
         anybyte = 0
@@ -1538,9 +1519,9 @@
             # If the incremental major collection is currently at
             # STATE_MARKING, then we must add to 'objects_to_trace' all
             # black objects that go through 'old_objects_pointing_to_young'.
+            # This basically turns them gray again.
             if state_is_marking and self.header(obj).tid & GCFLAG_VISITED != 0:
                 self.header(obj).tid &= ~GCFLAG_VISITED
-                self.header(obj).tid |= GCFLAG_GRAY
                 self.objects_to_trace.append(obj)
             #
             # Trace the 'obj' to replace pointers to nursery with pointers
@@ -1721,6 +1702,15 @@
             old.append(new.pop())
         new.delete()
 
+    def minor_and_major_collection(self):
+        # First, finish the current major gc, if there is one in progress.
+        # This is a no-op if the gc_state is already STATE_SCANNING.
+        self.gc_step_until(STATE_SCANNING)
+        #
+        # Then do a complete collection again.
+        self.gc_step_until(STATE_MARKING)
+        self.gc_step_until(STATE_SCANNING)
+
     def gc_step_until(self, state, reserving_size=0):
         while self.gc_state != state:
             self.minor_collection()
@@ -1854,16 +1844,16 @@
         size_gc_header = self.gcheaderbuilder.size_gc_header
         obj = hdr + size_gc_header
         if self.header(obj).tid & GCFLAG_VISITED:
-            self.header(obj).tid &= ~(GCFLAG_VISITED|GCFLAG_GRAY)
+            self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
         return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
-        self.header(obj).tid &= ~(GCFLAG_VISITED|GCFLAG_GRAY)
+        self.header(obj).tid &= ~GCFLAG_VISITED
 
     def free_rawmalloced_object_if_unvisited(self, obj):
         if self.header(obj).tid & GCFLAG_VISITED:
-            self.header(obj).tid &= ~(GCFLAG_VISITED|GCFLAG_GRAY)   # survives
+            self.header(obj).tid &= ~GCFLAG_VISITED   # survives
             self.old_rawmalloced_objects.append(obj)
         else:
             size_gc_header = self.gcheaderbuilder.size_gc_header
@@ -1889,14 +1879,16 @@
             self.rawmalloced_total_size -= r_uint(allocsize)
 
     def start_free_rawmalloc_objects(self):
+        ll_assert(not self.raw_malloc_might_sweep.non_empty(),
+                  "raw_malloc_might_sweep must be empty")
         (self.raw_malloc_might_sweep, self.old_rawmalloced_objects) = (
             self.old_rawmalloced_objects, self.raw_malloc_might_sweep)
 
     # Returns true when finished processing objects
     def free_unvisited_rawmalloc_objects_step(self, nobjects):
         while self.raw_malloc_might_sweep.non_empty() and nobjects > 0:
-            self.free_rawmalloced_object_if_unvisited(
-                                             self.raw_malloc_might_sweep.pop())
+            obj = self.raw_malloc_might_sweep.pop()
+            self.free_rawmalloced_object_if_unvisited(obj)
             nobjects -= 1
 
         return nobjects
@@ -2191,6 +2183,8 @@
     # The code relies on the fact that no weakref can be an old object
     # weakly pointing to a young object.  Indeed, weakrefs are immutable
     # so they cannot point to an object that was created after it.
+    # Thanks to this, during a minor collection, we don't have to fix
+    # or clear the address stored in old weakrefs.
     def invalidate_young_weakrefs(self):
         """Called during a nursery collection."""
         # walk over the list of objects that contain weakrefs and are in the

From noreply at buildbot.pypy.org  Fri Oct 11 19:21:59 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 19:21:59 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: fix
Message-ID: <20131011172159.ED2741C021C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67326:1d3f04da10c9
Date: 2013-10-11 19:05 +0200
http://bitbucket.org/pypy/pypy/changeset/1d3f04da10c9/

Log:	fix

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1785,7 +1785,7 @@
                                                  2 * limit + remaining)
             # XXX tweak the limits above
             #
-            if remaining == 0 and done:
+            if remaining > 0 and done:
                 self.num_major_collects += 1
                 #
                 # We also need to reset the GCFLAG_VISITED on prebuilt GC objects.

From noreply at buildbot.pypy.org  Fri Oct 11 19:22:01 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 19:22:01 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Progress
Message-ID: <20131011172201.26DB91C021C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67327:29b87bb8989c
Date: 2013-10-11 19:21 +0200
http://bitbucket.org/pypy/pypy/changeset/29b87bb8989c/

Log:	Progress

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1190,7 +1190,8 @@
             # we're going to call this function a lot of times for the
             # same object; moreover we'd need to pass the 'newvalue' as
             # an argument here.  The JIT has always called a
-            # 'newvalue'-less version, too.
+            # 'newvalue'-less version, too.  Moreover, the incremental
+            # GC nowadays relies on this fact.
             self.old_objects_pointing_to_young.append(addr_struct)
             objhdr = self.header(addr_struct)
             objhdr.tid &= ~GCFLAG_TRACK_YOUNG_PTRS
@@ -1518,9 +1519,10 @@
             #
             # If the incremental major collection is currently at
             # STATE_MARKING, then we must add to 'objects_to_trace' all
-            # black objects that go through 'old_objects_pointing_to_young'.
-            # This basically turns them gray again.
-            if state_is_marking and self.header(obj).tid & GCFLAG_VISITED != 0:
+            # objects that go through 'old_objects_pointing_to_young'.
+            # This basically turns black objects gray again, but also
+            # makes sure that we see otherwise-white objects.
+            if state_is_marking:
                 self.header(obj).tid &= ~GCFLAG_VISITED
                 self.objects_to_trace.append(obj)
             #
@@ -1728,7 +1730,7 @@
     # is done before every major collection step
     def major_collection_step(self, reserving_size=0):
         debug_start("gc-collect-step")
-        debug_print("stating gc state: ", GC_STATES[self.gc_state])
+        debug_print("starting gc state: ", GC_STATES[self.gc_state])
         # Debugging checks
         ll_assert(self.nursery_free == self.nursery,
                   "nursery not empty in major_collection_step()")

From noreply at buildbot.pypy.org  Fri Oct 11 20:04:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 20:04:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: Jit: avoid a guard "len(s2) != 0" in
 the common case, just to handle
Message-ID: <20131011180419.C838C1D22C4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67328:d994777be5ab
Date: 2013-10-11 20:03 +0200
http://bitbucket.org/pypy/pypy/changeset/d994777be5ab/

Log:	Jit: avoid a guard "len(s2) != 0" in the common case, just to handle
	the unusual case of find(s, '') or count(s, '').

diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -577,9 +577,7 @@
             return -1
 
         m = len(s2.chars)
-        if m == 0:
-            return start
-        elif m == 1:
+        if m == 1:
             return cls.ll_find_char(s1, s2.chars[0], start, end)
 
         return cls.ll_search(s1, s2, start, end, FAST_FIND)
@@ -594,9 +592,7 @@
             return -1
 
         m = len(s2.chars)
-        if m == 0:
-            return end
-        elif m == 1:
+        if m == 1:
             return cls.ll_rfind_char(s1, s2.chars[0], start, end)
 
         return cls.ll_search(s1, s2, start, end, FAST_RFIND)
@@ -611,9 +607,7 @@
             return 0
 
         m = len(s2.chars)
-        if m == 0:
-            return end - start + 1
-        elif m == 1:
+        if m == 1:
             return cls.ll_count_char(s1, s2.chars[0], start, end)
 
         res = cls.ll_search(s1, s2, start, end, FAST_COUNT)
@@ -629,6 +623,14 @@
         n = end - start
         m = len(s2.chars)
 
+        if m == 0:
+            if mode == FAST_COUNT:
+                return end - start + 1
+            elif mode == FAST_RFIND:
+                return end
+            else:
+                return start
+
         w = n - m
 
         if w < 0:

From noreply at buildbot.pypy.org  Fri Oct 11 20:38:11 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 20:38:11 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Argh,
 GCFLAG_VISITED is used for two different purposes during
Message-ID: <20131011183811.4D2FB1D22C5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67329:5484c432fa85
Date: 2013-10-11 20:29 +0200
http://bitbucket.org/pypy/pypy/changeset/5484c432fa85/

Log:	Argh, GCFLAG_VISITED is used for two different purposes during minor
	and major collection --- but now major collection is longer.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -98,8 +98,7 @@
 # 'prebuilt_root_objects'.
 GCFLAG_NO_HEAP_PTRS = first_gcflag << 1
 
-# The following flag is set on surviving objects during a major collection,
-# and on surviving raw-malloced young objects during a minor collection.
+# The following flag is set on surviving objects during a major collection.
 GCFLAG_VISITED      = first_gcflag << 2
 
 # The following flag is set on nursery objects of which we asked the id
@@ -124,7 +123,11 @@
 # note that GCFLAG_CARDS_SET is the most significant bit of a byte:
 # this is required for the JIT (x86)
 
-_GCFLAG_FIRST_UNUSED = first_gcflag << 8    # the first unused bit
+# The following flag is set on surviving raw-malloced young objects during
+# a minor collection.
+GCFLAG_VISITED_RMY   = first_gcflag << 8
+
+_GCFLAG_FIRST_UNUSED = first_gcflag << 9    # the first unused bit
 
 
 # States for the incremental GC
@@ -734,7 +737,7 @@
             raise MemoryError
         #
         # If somebody calls this function a lot, we must eventually
-        # force a full collection.
+        # force a full collection.  XXX make this more incremental!
         if (float(self.get_total_memory_used()) + raw_malloc_usage(totalsize) >
                 self.next_major_collection_threshold):
             self.gc_step_until(STATE_SWEEPING)
@@ -1033,6 +1036,8 @@
         # collection step.  No object should be in the nursery
         ll_assert(not self.is_in_nursery(obj),
                   "object in nursery after collection")
+        ll_assert(self.header(obj).tid & GCFLAG_VISITED_RMY == 0,
+                  "GCFLAG_VISITED_RMY after collection")
 
         if self.gc_state == STATE_SCANNING:
             self._debug_check_object_scanning(obj)
@@ -1365,7 +1370,7 @@
         #
         # First, find the roots that point to young objects.  All nursery
         # objects found are copied out of the nursery, and the occasional
-        # young raw-malloced object is flagged with GCFLAG_VISITED.
+        # young raw-malloced object is flagged with GCFLAG_VISITED_RMY.
         # Note that during this step, we ignore references to further
         # young objects; only objects directly referenced by roots
         # are copied out or flagged.  They are also added to the list
@@ -1381,7 +1386,8 @@
             # Now trace objects from 'old_objects_pointing_to_young'.
             # All nursery objects they reference are copied out of the
             # nursery, and again added to 'old_objects_pointing_to_young'.
-            # All young raw-malloced object found are flagged GCFLAG_VISITED.
+            # All young raw-malloced object found are flagged
+            # GCFLAG_VISITED_RMY.
             # We proceed until 'old_objects_pointing_to_young' is empty.
             self.collect_oldrefs_to_nursery()
             #
@@ -1555,7 +1561,7 @@
         #print '_trace_drag_out(%x: %r)' % (hash(obj.ptr._obj), obj)
         #
         # If 'obj' is not in the nursery, nothing to change -- expect
-        # that we must set GCFLAG_VISITED on young raw-malloced objects.
+        # that we must set GCFLAG_VISITED_RMY on young raw-malloced objects.
         if not self.is_in_nursery(obj):
             # cache usage trade-off: I think that it is a better idea to
             # check if 'obj' is in young_rawmalloced_objects with an access
@@ -1628,15 +1634,15 @@
     def _visit_young_rawmalloced_object(self, obj):
         # 'obj' points to a young, raw-malloced object.
         # Any young rawmalloced object never seen by the code here
-        # will end up without GCFLAG_VISITED, and be freed at the
+        # will end up without GCFLAG_VISITED_RMY, and be freed at the
         # end of the current minor collection.  Note that there was
         # a bug in which dying young arrays with card marks would
         # still be scanned before being freed, keeping a lot of
         # objects unnecessarily alive.
         hdr = self.header(obj)
-        if hdr.tid & GCFLAG_VISITED:
+        if hdr.tid & GCFLAG_VISITED_RMY:
             return
-        hdr.tid |= GCFLAG_VISITED
+        hdr.tid |= GCFLAG_VISITED_RMY
         #
         # we just made 'obj' old, so we need to add it to the correct lists
         added_somewhere = False
@@ -1687,9 +1693,9 @@
         self.young_rawmalloced_objects = self.null_address_dict()
 
     def _free_young_rawmalloced_obj(self, obj, ignored1, ignored2):
-        # If 'obj' has GCFLAG_VISITED, it was seen by _trace_drag_out
+        # If 'obj' has GCFLAG_VISITED_RMY, it was seen by _trace_drag_out
         # and survives.  Otherwise, it dies.
-        self.free_rawmalloced_object_if_unvisited(obj)
+        self.free_rawmalloced_object_if_unvisited(obj, GCFLAG_VISITED_RMY)
 
     def remove_young_arrays_from_old_objects_pointing_to_young(self):
         old = self.old_objects_pointing_to_young
@@ -1853,9 +1859,9 @@
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
 
-    def free_rawmalloced_object_if_unvisited(self, obj):
-        if self.header(obj).tid & GCFLAG_VISITED:
-            self.header(obj).tid &= ~GCFLAG_VISITED   # survives
+    def free_rawmalloced_object_if_unvisited(self, obj, check_flag):
+        if self.header(obj).tid & check_flag:
+            self.header(obj).tid &= ~check_flag   # survives
             self.old_rawmalloced_objects.append(obj)
         else:
             size_gc_header = self.gcheaderbuilder.size_gc_header
@@ -1890,7 +1896,7 @@
     def free_unvisited_rawmalloc_objects_step(self, nobjects):
         while self.raw_malloc_might_sweep.non_empty() and nobjects > 0:
             obj = self.raw_malloc_might_sweep.pop()
-            self.free_rawmalloced_object_if_unvisited(obj)
+            self.free_rawmalloced_object_if_unvisited(obj, GCFLAG_VISITED)
             nobjects -= 1
 
         return nobjects
@@ -2210,7 +2216,7 @@
             elif (bool(self.young_rawmalloced_objects) and
                   self.young_rawmalloced_objects.contains(pointing_to)):
                 # young weakref to a young raw-malloced object
-                if self.header(pointing_to).tid & GCFLAG_VISITED:
+                if self.header(pointing_to).tid & GCFLAG_VISITED_RMY:
                     pass    # survives, but does not move
                 else:
                     (obj + offset).address[0] = llmemory.NULL

From noreply at buildbot.pypy.org  Fri Oct 11 20:38:12 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 11 Oct 2013 20:38:12 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Fix
Message-ID: <20131011183812.80C791D22C5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67330:af229eaaa2ca
Date: 2013-10-11 20:37 +0200
http://bitbucket.org/pypy/pypy/changeset/af229eaaa2ca/

Log:	Fix

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1629,6 +1629,13 @@
         if self.has_gcptr(typeid):
             # we only have to do it if we have any gcptrs
             self.old_objects_pointing_to_young.append(newobj)
+        else:
+            # we don't need to add this to 'old_objects_pointing_to_young',
+            # but in the STATE_MARKING phase we still need this bit...
+            if self.gc_state == STATE_MARKING:
+                self.header(newobj).tid &= ~GCFLAG_VISITED
+                self.objects_to_trace.append(newobj)
+
     _trace_drag_out._always_inline_ = True
 
     def _visit_young_rawmalloced_object(self, obj):

From noreply at buildbot.pypy.org  Sat Oct 12 01:29:23 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 12 Oct 2013 01:29:23 +0200 (CEST)
Subject: [pypy-commit] pypy remove-intlong-smm: merge default
Message-ID: <20131011232923.2CF561D22C5@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: remove-intlong-smm
Changeset: r67332:3cbdd1b77142
Date: 2013-10-11 16:26 -0700
http://bitbucket.org/pypy/pypy/changeset/3cbdd1b77142/

Log:	merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -52,6 +52,9 @@
 .. branch: ndarray-subtype
 Allow subclassing ndarray, i.e. matrix
 
+.. branch: ndarray-sort
+Implement ndarray in-place sorting (for numeric types, no non-native byte order)
+
 .. branch: pypy-pyarray
 Implement much of numpy's c api in cpyext, allows (slow) access to ndarray
 from c
@@ -87,6 +90,8 @@
 .. branch: no-release-gil
 .. branch: safe-win-mmap
 .. branch: boolean-indexing-cleanup
+.. branch: cpyext-best_base
+.. branch: fileops2
 
 .. branch: nobold-backtrace
 Work on improving UnionError messages and stack trace displays.
@@ -103,3 +108,5 @@
 
 .. branch: file-support-in-rpython
 make open() and friends rpython
+
+
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -99,7 +99,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii'])
+                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
@@ -196,7 +196,7 @@
         assert PyUnicode_GetDefaultEncoding() == 'ascii'
 
 class AppTestCpythonExtensionBase(LeakCheckingTest):
-    
+
     def setup_class(cls):
         cls.space.getbuiltinmodule("cpyext")
         from pypy.module.imp.importing import importhook
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -156,7 +156,7 @@
             def __init__(self):
                 self.foobar = 32
                 super(UnicodeSubclass2, self).__init__()
-        
+
         newobj = UnicodeSubclass2()
         assert newobj.get_val() == 42
         assert newobj.foobar == 32
@@ -358,6 +358,13 @@
         assert w_obj is None
         assert api.PyErr_Occurred() is None
 
+    def test_ndarray_ref(self, space, api):
+        w_obj = space.appexec([], """():
+            import numpypy as np
+            return np.int64(2)""")
+        ref = make_ref(space, w_obj)
+        api.Py_DecRef(ref)
+
 class AppTestSlots(AppTestCpythonExtensionBase):
     def test_some_slots(self):
         module = self.import_extension('foo', [
@@ -525,7 +532,7 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
-        
+
     def test_bool(self):
         module = self.import_extension('foo', [
             ("newInt", "METH_VARARGS",
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -4,7 +4,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.annlowlevel import llhelper
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
-from pypy.objspace.std.typeobject import W_TypeObject
+from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
@@ -574,33 +574,7 @@
 def best_base(space, bases_w):
     if not bases_w:
         return None
-
-    w_winner = None
-    w_base = None
-    for w_base_i in bases_w:
-        if isinstance(w_base_i, W_ClassObject):
-            # old-style base
-            continue
-        assert isinstance(w_base_i, W_TypeObject)
-        w_candidate = solid_base(space, w_base_i)
-        if not w_winner:
-            w_winner = w_candidate
-            w_base = w_base_i
-        elif space.abstract_issubclass_w(w_winner, w_candidate):
-            pass
-        elif space.abstract_issubclass_w(w_candidate, w_winner):
-            w_winner = w_candidate
-            w_base = w_base_i
-        else:
-            raise OperationError(
-                space.w_TypeError,
-                space.wrap("multiple bases have instance lay-out conflict"))
-    if w_base is None:
-        raise OperationError(
-            space.w_TypeError,
-                space.wrap("a new-style class can't have only classic bases"))
-
-    return w_base
+    return find_best_base(space, bases_w)
 
 def inherit_slots(space, pto, w_base):
     # XXX missing: nearly everything
diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -356,6 +356,10 @@
         from pypy.module.micronumpy.arrayimpl.sort import argsort_array
         return argsort_array(self, space, w_axis)
 
+    def sort(self, space, w_axis, w_order):
+        from pypy.module.micronumpy.arrayimpl.sort import sort_array
+        return sort_array(self, space, w_axis, w_order)
+
     def base(self):
         return None
 
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -17,7 +17,7 @@
 
 INT_SIZE = rffi.sizeof(lltype.Signed)
 
-def make_sort_function(space, itemtype, comp_type, count=1):
+def make_argsort_function(space, itemtype, comp_type, count=1):
     TP = itemtype.T
     step = rffi.sizeof(TP)
 
@@ -137,8 +137,8 @@
         else:
             shape = arr.get_shape()
             if axis < 0:
-                axis = len(shape) + axis - 1
-            if axis < 0 or axis > len(shape):
+                axis = len(shape) + axis
+            if axis < 0 or axis >= len(shape):
                 raise OperationError(space.w_IndexError, space.wrap(
                                                     "Wrong axis %d" % axis))
             iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
@@ -162,7 +162,7 @@
     return argsort
 
 def argsort_array(arr, space, w_axis):
-    cache = space.fromcache(SortCache) # that populates SortClasses
+    cache = space.fromcache(ArgSortCache) # that populates ArgSortClasses
     itemtype = arr.dtype.itemtype
     for tp in all_types:
         if isinstance(itemtype, tp[0]):
@@ -178,6 +178,166 @@
 all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
 all_types = unrolling_iterable(all_types)
 
+def make_sort_function(space, itemtype, comp_type, count=1):
+    TP = itemtype.T
+    step = rffi.sizeof(TP)
+
+    class Repr(object):
+        def __init__(self, stride_size, size, values, start):
+            self.stride_size = stride_size
+            self.start = start
+            self.size = size
+            self.values = values
+
+        def getitem(self, item):
+            if count < 2:
+                v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                                    + self.start)
+            else:
+                v = []
+                for i in range(count):
+                    _v = raw_storage_getitem(TP, self.values, item * self.stride_size
+                                    + self.start + step * i)
+                    v.append(_v)
+            if comp_type == 'int':
+                v = intmask(v)
+            elif comp_type == 'float':
+                v = float(v)
+            elif comp_type == 'complex':
+                v = [float(v[0]),float(v[1])]
+            else:
+                raise NotImplementedError('cannot reach')
+            return (v)
+
+        def setitem(self, idx, item):
+            if count < 2:
+                raw_storage_setitem(self.values, idx * self.stride_size +
+                                self.start, rffi.cast(TP, item))
+            else:
+                i = 0
+                for val in item:
+                    raw_storage_setitem(self.values, idx * self.stride_size +
+                                self.start + i*step, rffi.cast(TP, val))
+                    i += 1
+
+    class ArgArrayRepWithStorage(Repr):
+        def __init__(self, stride_size, size):
+            start = 0
+            values = alloc_raw_storage(size * stride_size,
+                                            track_allocation=False)
+            Repr.__init__(self, stride_size,
+                          size, values, start)
+
+        def __del__(self):
+            free_raw_storage(self.values, track_allocation=False)
+
+    def arg_getitem(lst, item):
+        return lst.getitem(item)
+
+    def arg_setitem(lst, item, value):
+        lst.setitem(item, value)
+
+    def arg_length(lst):
+        return lst.size
+
+    def arg_getitem_slice(lst, start, stop):
+        retval = ArgArrayRepWithStorage(lst.stride_size, stop-start)
+        for i in range(stop-start):
+            retval.setitem(i, lst.getitem(i+start))
+        return retval
+
+    if count < 2:
+        def arg_lt(a, b):
+            # handles NAN and INF
+            return a < b or b != b and a == a
+    else:
+        def arg_lt(a, b):
+            for i in range(count):
+                if b[i] != b[i] and a[i] == a[i]:
+                    return True
+                elif b[i] == b[i] and a[i] != a[i]:
+                    return False
+            for i in range(count):
+                if a[i] < b[i]:
+                    return True
+                elif a[i] > b[i]:
+                    return False
+            # Does numpy do True?
+            return False
+
+    ArgSort = make_timsort_class(arg_getitem, arg_setitem, arg_length,
+                                 arg_getitem_slice, arg_lt)
+
+    def sort(arr, space, w_axis, itemsize):
+        if w_axis is space.w_None:
+            # note that it's fine to pass None here as we're not going
+            # to pass the result around (None is the link to base in slices)
+            arr = arr.reshape(space, None, [arr.get_size()])
+            axis = 0
+        elif w_axis is None:
+            axis = -1
+        else:
+            axis = space.int_w(w_axis)
+        # create array of indexes
+        if len(arr.get_shape()) == 1:
+            r = Repr(itemsize, arr.get_size(), arr.get_storage(),
+                     arr.start)
+            ArgSort(r).sort()
+        else:
+            shape = arr.get_shape()
+            if axis < 0:
+                axis = len(shape) + axis
+            if axis < 0 or axis >= len(shape):
+                raise OperationError(space.w_IndexError, space.wrap(
+                                                    "Wrong axis %d" % axis))
+            iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
+            iter = AxisIterator(arr, iterable_shape, axis, False)
+            stride_size = arr.strides[axis]
+            axis_size = arr.shape[axis]
+            while not iter.done():
+                r = Repr(stride_size, axis_size, arr.get_storage(), iter.offset)
+                ArgSort(r).sort()
+                iter.next()
+
+    return sort
+
+def sort_array(arr, space, w_axis, w_order):
+    cache = space.fromcache(SortCache) # that populates SortClasses
+    itemtype = arr.dtype.itemtype
+    if not arr.dtype.native:
+        raise OperationError(space.w_NotImplementedError,
+           space.wrap("sorting of non-native btyeorder not supported yet"))
+    for tp in all_types:
+        if isinstance(itemtype, tp[0]):
+            return cache._lookup(tp)(arr, space, w_axis,
+                                     itemtype.get_element_size())
+    # XXX this should probably be changed
+    raise OperationError(space.w_NotImplementedError,
+           space.wrap("sorting of non-numeric types " + \
+                  "'%s' is not implemented" % arr.dtype.get_name(), ))
+
+all_types = (types.all_float_types + types.all_complex_types +
+             types.all_int_types)
+all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
+all_types = unrolling_iterable(all_types)
+
+class ArgSortCache(object):
+    built = False
+
+    def __init__(self, space):
+        if self.built:
+            return
+        self.built = True
+        cache = {}
+        for cls, it in all_types._items:
+            if it == 'complex':
+                cache[cls] = make_argsort_function(space, cls, it, 2)
+            else:
+                cache[cls] = make_argsort_function(space, cls, it)
+        self.cache = cache
+        self._lookup = specialize.memo()(lambda tp : cache[tp[0]])
+
+
 class SortCache(object):
     built = False
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -629,9 +629,13 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "setflags not implemented yet"))
 
-    def descr_sort(self, space, w_axis=-1, w_kind='quicksort', w_order=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "sort not implemented yet"))
+    @unwrap_spec(kind=str)
+    def descr_sort(self, space, w_axis=None, kind='quicksort', w_order=None):
+        # happily ignore the kind
+        # modify the array in-place
+        if self.is_scalar():
+            return
+        return self.implementation.sort(space, w_axis, w_order)
 
     def descr_squeeze(self, space):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -1118,6 +1122,7 @@
     conj = interp2app(W_NDimArray.descr_conj),
 
     argsort  = interp2app(W_NDimArray.descr_argsort),
+    sort  = interp2app(W_NDimArray.descr_sort),
     astype   = interp2app(W_NDimArray.descr_astype),
     base     = GetSetProperty(W_NDimArray.descr_get_base),
     byteswap = interp2app(W_NDimArray.descr_byteswap),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2652,55 +2652,6 @@
         assert array([1, 2, 3], '>i2')[::2].tostring() == '\x00\x01\x00\x03'
         assert array(0, dtype='i2').tostring() == '\x00\x00'
 
-    def test_argsort_dtypes(self):
-        from numpypy import array, arange
-        assert array(2.0).argsort() == 0
-        nnp = self.non_native_prefix
-        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
-                        nnp + 'i2', complex]:
-            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
-            c = a.copy()
-            res = a.argsort()
-            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
-                'a,res,dtype %r,%r,%r' % (a,res,dtype)
-            assert (a == c).all() # not modified
-            a = arange(100)
-            assert (a.argsort() == a).all()
-        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
-
-    def test_argsort_nd(self):
-        from numpypy import array
-        a = array([[4, 2], [1, 3]])
-        assert (a.argsort() == [[1, 0], [0, 1]]).all()
-        a = array(range(10) + range(10) + range(10))
-        b = a.argsort()
-        assert (b[:3] == [0, 10, 20]).all()
-        #trigger timsort 'run' mode which calls arg_getitem_slice
-        a = array(range(100) + range(100) + range(100))
-        b = a.argsort()
-        assert (b[:3] == [0, 100, 200]).all()
-        a = array([[[]]]).reshape(3,4,0)
-        b = a.argsort()
-        assert b.size == 0
-
-    def test_argsort_random(self):
-        from numpypy import array
-        from _random import Random
-        rnd = Random(1)
-        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
-        a.argsort()
-
-    def test_argsort_axis(self):
-        from numpypy import array
-        a = array([[4, 2], [1, 3]])
-        assert (a.argsort(axis=None) == [2, 1, 3, 0]).all()
-        assert (a.argsort(axis=-1) == [[1, 0], [0, 1]]).all()
-        assert (a.argsort(axis=0) == [[1, 0], [0, 1]]).all()
-        assert (a.argsort(axis=1) == [[1, 0], [0, 1]]).all()
-        a = array([[3, 2, 1], [1, 2, 3]])
-        assert (a.argsort(axis=0) == [[1, 0, 0], [0, 1, 1]]).all()
-        assert (a.argsort(axis=1) == [[2, 1, 0], [0, 1, 2]]).all()
-
 
 class AppTestRanges(BaseNumpyAppTest):
     def test_arange(self):
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -0,0 +1,322 @@
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestSupport(BaseNumpyAppTest):
+    def setup_class(cls):
+        import struct
+        BaseNumpyAppTest.setup_class.im_func(cls)
+        cls.w_data = cls.space.wrap(struct.pack('dddd', 1, 2, 3, 4))
+        cls.w_fdata = cls.space.wrap(struct.pack('f', 2.3))
+        cls.w_float16val = cls.space.wrap('\x00E') # 5.0 in float16
+        cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
+        cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
+        cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+
+    def test_argsort_dtypes(self):
+        from numpypy import array, arange
+        assert array(2.0).argsort() == 0
+        nnp = self.non_native_prefix
+        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
+                        nnp + 'i2', complex]:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            c = a.copy()
+            res = a.argsort()
+            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
+                'a,res,dtype %r,%r,%r' % (a,res,dtype)
+            assert (a == c).all() # not modified
+            a = arange(100)
+            assert (a.argsort() == a).all()
+        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
+
+    def test_argsort_nd(self):
+        from numpypy import array
+        a = array([[4, 2], [1, 3]])
+        assert (a.argsort() == [[1, 0], [0, 1]]).all()
+        a = array(range(10) + range(10) + range(10))
+        b = a.argsort()
+        assert (b[:3] == [0, 10, 20]).all()
+        #trigger timsort 'run' mode which calls arg_getitem_slice
+        a = array(range(100) + range(100) + range(100))
+        b = a.argsort()
+        assert (b[:3] == [0, 100, 200]).all()
+        a = array([[[]]]).reshape(3,4,0)
+        b = a.argsort()
+        assert b.size == 0
+
+    def test_argsort_random(self):
+        from numpypy import array
+        from _random import Random
+        rnd = Random(1)
+        a = array([rnd.random() for i in range(512*2)]).reshape(512,2)
+        a.argsort()
+
+    def test_argsort_axis(self):
+        from numpypy import array
+        a = array([[4, 2], [1, 3]])
+        assert (a.argsort(axis=None) == [2, 1, 3, 0]).all()
+        assert (a.argsort(axis=-1) == [[1, 0], [0, 1]]).all()
+        assert (a.argsort(axis=0) == [[1, 0], [0, 1]]).all()
+        assert (a.argsort(axis=1) == [[1, 0], [0, 1]]).all()
+        a = array([[3, 2, 1], [1, 2, 3]])
+        assert (a.argsort(axis=0) == [[1, 0, 0], [0, 1, 1]]).all()
+        assert (a.argsort(axis=1) == [[2, 1, 0], [0, 1, 2]]).all()
+
+    def test_sort_dtypes(self):
+        from numpypy import array, arange
+        for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
+                        'i2', complex]:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
+            c = a.copy()
+            a.sort()
+            assert (a == b).all(), \
+                'a,orig,dtype %r,%r,%r' % (a,c,dtype)
+        a = arange(100)
+        c = a.copy()
+        a.sort()
+        assert (a == c).all()
+
+    def test_sort_dtypesi_nonnative(self):
+        from numpypy import array
+        nnp = self.non_native_prefix
+        for dtype in [ nnp + 'i2']:
+            a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
+            c = a.copy()
+            exc = raises(NotImplementedError, a.sort)
+            assert exc.value[0].find('supported') >= 0
+            #assert (a == b).all(), \
+            #    'a,orig,dtype %r,%r,%r' % (a,c,dtype)
+
+
+# tests from numpy/tests/test_multiarray.py
+    def test_sort_corner_cases(self):
+        # test ordering for floats and complex containing nans. It is only
+        # necessary to check the lessthan comparison, so sorts that
+        # only follow the insertion sort path are sufficient. We only
+        # test doubles and complex doubles as the logic is the same.
+
+        # check doubles
+        from numpypy import array, nan, zeros, complex128, arange
+        from numpy import isnan
+        a = array([nan, 1, 0])
+        b = a.copy()
+        b.sort()
+        assert (isnan(b) == isnan(a[::-1])).all()
+        assert (b[:2] == a[::-1][:2]).all()
+
+        # check complex
+        a = zeros(9, dtype=complex128)
+        a.real += [nan, nan, nan, 1, 0, 1, 1, 0, 0]
+        a.imag += [nan, 1, 0, nan, nan, 1, 0, 1, 0]
+        b = a.copy()
+        b.sort()
+        assert (isnan(b) == isnan(a[::-1])).all()
+        assert (b[:4] == a[::-1][:4]).all()
+
+        # all c scalar sorts use the same code with different types
+        # so it suffices to run a quick check with one type. The number
+        # of sorted items must be greater than ~50 to check the actual
+        # algorithm because quick and merge sort fall over to insertion
+        # sort for small arrays.
+        a = arange(101)
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h'] :
+            msg = "scalar sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+        # test complex sorts. These use the same code as the scalars
+        # but the compare fuction differs.
+        ai = a*1j + 1
+        bi = b*1j + 1
+        for kind in ['q', 'm', 'h'] :
+            msg = "complex sort, real part == 1, kind=%s" % kind
+            c = ai.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+            c = bi.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+        ai = a + 1j
+        bi = b + 1j
+        for kind in ['q', 'm', 'h'] :
+            msg = "complex sort, imag part == 1, kind=%s" % kind
+            c = ai.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+            c = bi.copy();
+            c.sort(kind=kind)
+            assert (c == ai).all(), msg
+
+        # check axis handling. This should be the same for all type
+        # specific sorts, so we only check it for one type and one kind
+        a = array([[3, 2], [1, 0]])
+        b = array([[1, 0], [3, 2]])
+        c = array([[2, 3], [0, 1]])
+        d = a.copy()
+        d.sort(axis=0)
+        assert (d == b).all(), "test sort with axis=0"
+        d = a.copy()
+        d.sort(axis=1)
+        assert (d == c).all(), "test sort with axis=1"
+        d = a.copy()
+        d.sort()
+        assert (d == c).all(), "test sort with default axis"
+
+    def test_sort_corner_cases_string_records(self):
+        skip('not implemented yet')
+        from numpypy import array, dtype
+        # test string sorts.
+        s = 'aaaaaaaa'
+        a = array([s + chr(i) for i in range(101)])
+        b = a[::-1].copy()
+        for kind in ['q', 'm', 'h'] :
+            msg = "string sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+
+        # test record array sorts.
+        dt =dtype([('f', float), ('i', int)])
+        a = array([(i, i) for i in range(101)], dtype = dt)
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "object sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_unicode(self):
+        from numpypy import array
+        # test unicode sorts.
+        s = 'aaaaaaaa'
+        try:
+            a = array([s + chr(i) for i in range(101)], dtype=unicode)
+            b = a[::-1].copy()
+        except:
+            skip('unicode type not supported yet')
+        for kind in ['q', 'm', 'h'] :
+            msg = "unicode sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_objects(self):
+        # test object array sorts.
+        from numpypy import empty
+        try:
+            a = empty((101,), dtype=object)
+        except:
+            skip('object type not supported yet')
+        a[:] = list(range(101))
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "object sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_datetime(self):
+        from numpypy import arange
+        # test datetime64 sorts.
+        try:
+            a = arange(0, 101, dtype='datetime64[D]')
+        except:
+            skip('datetime type not supported yet')
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "datetime64 sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+        # test timedelta64 sorts.
+        a = arange(0, 101, dtype='timedelta64[D]')
+        b = a[::-1]
+        for kind in ['q', 'h', 'm'] :
+            msg = "timedelta64 sort, kind=%s" % kind
+            c = a.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+            c = b.copy();
+            c.sort(kind=kind)
+            assert (c == a).all(), msg
+
+    def test_sort_order(self):
+        from numpypy import array, zeros
+        from sys import byteorder
+        # Test sorting an array with fields
+        skip('not implemented yet')
+        x1 = array([21, 32, 14])
+        x2 = array(['my', 'first', 'name'])
+        x3=array([3.1, 4.5, 6.2])
+        r=zeros(3, dtype=[('id','i'),('word','S5'),('number','f')])
+        r['id'] = x1
+        r['word'] = x2
+        r['number'] = x3
+
+        r.sort(order=['id'])
+        assert (r['id'] == [14, 21, 32]).all()
+        assert (r['word'] == ['name', 'my', 'first']).all()
+        assert max(abs(r['number'] - [6.2, 3.1, 4.5])) < 1e-6
+
+        r.sort(order=['word'])
+        assert (r['id'] == [32, 21, 14]).all()
+        assert (r['word'] == ['first', 'my', 'name']).all()
+        assert max(abs(r['number'] - [4.5, 3.1, 6.2])) < 1e-6
+
+        r.sort(order=['number'])
+        assert (r['id'] == [21, 32, 14]).all()
+        assert (r['word'] == ['my', 'first', 'name']).all()
+        assert max(abs(r['number'] - [3.1, 4.5, 6.2])) < 1e-6
+
+        if byteorder == 'little':
+            strtype = '>i2'
+        else:
+            strtype = '<i2'
+        mydtype = [('name', 'S5'), ('col2', strtype)]
+        r = array([('a', 1), ('b', 255), ('c', 3), ('d', 258)],
+                     dtype= mydtype)
+        r.sort(order='col2')
+        assert (r['col2'] == [1, 3, 255, 258]).all()
+        assert (r == array([('a', 1), ('c', 3), ('b', 255), ('d', 258)],
+                                 dtype=mydtype)).all()
+
+
+# tests from numpy/tests/test_regression.py
+    def test_sort_bigendian(self):
+        skip('not implemented yet')
+        from numpypy import array, dtype
+        a = array(range(11),dtype='float64')
+        c = a.astype(dtype('<f8'))
+        c.sort()
+        assert max(abs(a-c)) < 1e-32
+
+    def test_string_sort_with_zeros(self):
+        skip('not implemented yet')
+        from numpypy import fromstring
+        """Check sort for strings containing zeros."""
+        x = fromstring("\x00\x02\x00\x01", dtype="S2")
+        y = fromstring("\x00\x01\x00\x02", dtype="S2")
+        x.sort(kind='q')
+        assert (x == y).all()
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1864,7 +1864,6 @@
         return interp_boxes.W_StringBox(arr,  0, arr.dtype)
 
     def fill(self, storage, width, box, start, stop, offset):
-        from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArrayNotOwning
         for i in xrange(start, stop, width):
             self._store(storage, i, offset, box)
 
@@ -1873,6 +1872,13 @@
 class UnicodeType(BaseType, BaseStringType):
     T = lltype.UniChar
 
+    @jit.unroll_safe
+    def coerce(self, space, dtype, w_item):
+        if isinstance(w_item, interp_boxes.W_UnicodeBox):
+            return w_item
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "coerce (probably from set_item) not implemented for unicode type"))
+
 NonNativeUnicodeType = UnicodeType
 
 class VoidType(BaseType, BaseStringType):
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1201,7 +1201,6 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
-            assert self.card_page_indices > 0
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
@@ -1236,6 +1235,7 @@
 
     def manually_copy_card_bits(self, source_addr, dest_addr, length):
         # manually copy the individual card marks from source to dest
+        assert self.card_page_indices > 0
         bytes = self.card_marking_bytes_for_length(length)
         #
         anybyte = 0
diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py
--- a/rpython/rlib/rdtoa.py
+++ b/rpython/rlib/rdtoa.py
@@ -217,13 +217,13 @@
 
         if exp >= 0:
             exp_str = str(exp)
-            if len(exp_str) < 2:
+            if len(exp_str) < 2 and not (flags & rfloat.DTSF_CUT_EXP_0):
                 s += e + '+0' + exp_str
             else:
                 s += e + '+' + exp_str
         else:
             exp_str = str(-exp)
-            if len(exp_str) < 2:
+            if len(exp_str) < 2 and not (flags & rfloat.DTSF_CUT_EXP_0):
                 s += e + '-0' + exp_str
             else:
                 s += e + '-' + exp_str
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -69,6 +69,7 @@
 DTSF_SIGN      = 0x1
 DTSF_ADD_DOT_0 = 0x2
 DTSF_ALT       = 0x4
+DTSF_CUT_EXP_0 = 0x8
 
 DIST_FINITE   = 1
 DIST_NAN      = 2
diff --git a/rpython/rlib/test/test_rdtoa.py b/rpython/rlib/test/test_rdtoa.py
--- a/rpython/rlib/test/test_rdtoa.py
+++ b/rpython/rlib/test/test_rdtoa.py
@@ -29,3 +29,7 @@
 def test_dtoa_precision():
     assert dtoa(1.1, code='f', precision=2) == "1.10"
     assert dtoa(1e12, code='g', precision=12) == "1e+12"
+
+def test_flag_cut_exp_0():
+    assert dtoa(1.1e9, code="g", precision=2, flags=rfloat.DTSF_CUT_EXP_0) == "1.1e+9"
+    assert dtoa(1.1e-9, code="g", precision=2, flags=rfloat.DTSF_CUT_EXP_0) == "1.1e-9"
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -820,8 +820,9 @@
             entry = entries[i]
             hash = entries.hash(i)
             key = entry.key
+            value = entry.value
             j = ll_dict_lookup(dic1, key, hash)
-            _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j)
+            _ll_dict_setitem_lookup_done(dic1, key, value, hash, j)
         i += 1
 ll_update.oopspec = 'dict.update(dic1, dic2)'
 
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -577,9 +577,7 @@
             return -1
 
         m = len(s2.chars)
-        if m == 0:
-            return start
-        elif m == 1:
+        if m == 1:
             return cls.ll_find_char(s1, s2.chars[0], start, end)
 
         return cls.ll_search(s1, s2, start, end, FAST_FIND)
@@ -594,9 +592,7 @@
             return -1
 
         m = len(s2.chars)
-        if m == 0:
-            return end
-        elif m == 1:
+        if m == 1:
             return cls.ll_rfind_char(s1, s2.chars[0], start, end)
 
         return cls.ll_search(s1, s2, start, end, FAST_RFIND)
@@ -611,9 +607,7 @@
             return 0
 
         m = len(s2.chars)
-        if m == 0:
-            return end - start + 1
-        elif m == 1:
+        if m == 1:
             return cls.ll_count_char(s1, s2.chars[0], start, end)
 
         res = cls.ll_search(s1, s2, start, end, FAST_COUNT)
@@ -629,6 +623,14 @@
         n = end - start
         m = len(s2.chars)
 
+        if m == 0:
+            if mode == FAST_COUNT:
+                return end - start + 1
+            elif mode == FAST_RFIND:
+                return end
+            else:
+                return start
+
         w = n - m
 
         if w < 0:
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -3,6 +3,7 @@
 import py
 
 from rpython.flowspace.model import summary
+from rpython.annotator.model import AnnotatorError
 from rpython.rtyper.lltypesystem.lltype import typeOf, Signed, malloc
 from rpython.rtyper.lltypesystem.rstr import LLHelpers, STR
 from rpython.rtyper.rstr import AbstractLLHelpers
@@ -361,16 +362,16 @@
             res = self.interpret(fn, [i, j])
             assert res == fn(i, j)
 
-    def test_find_TyperError(self):
+    def test_find_AnnotatorError(self):
         const = self.const
         def f():
             s = const('abc')
             s.find(s, 0, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
         def f():
             s = const('abc')
             s.find(s, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
 
     def test_find_empty_string(self):
         const = self.const
@@ -420,9 +421,8 @@
         const = self.const
         def f(i):
             return const("abc").rfind(const(''), i)
-        e = py.test.raises(TyperError, self.interpret, f, [-5])
-        assert str(e.value).startswith(
-            'str.rfind() start must be proven non-negative')
+        e = py.test.raises(AnnotatorError, self.interpret, f, [-5])
+        assert "rfind: not proven to have non-negative start" in str(e.value)
 
     def test_find_char(self):
         const = self.const
@@ -900,16 +900,16 @@
         res = self.interpret(fn, [])
         assert res == 1
 
-    def test_count_TyperError(self):
+    def test_count_AnnotatorError(self):
         const = self.const
         def f():
             s = const('abc')
             s.count(s, 0, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
         def f():
             s = const('abc')
             s.count(s, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
 
     def test_getitem_exc(self):
         const = self.const

From noreply at buildbot.pypy.org  Sat Oct 12 01:29:20 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 12 Oct 2013 01:29:20 +0200 (CEST)
Subject: [pypy-commit] pypy remove-intlong-smm: merge default
Message-ID: <20131011232920.C6A6F1D22C4@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: remove-intlong-smm
Changeset: r67331:5558c7f4430d
Date: 2013-10-09 17:27 -0700
http://bitbucket.org/pypy/pypy/changeset/5558c7f4430d/

Log:	merge default

diff too long, truncating to 2000 out of 3119 lines

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -363,9 +363,11 @@
     pass
 
 
-def connect(database, **kwargs):
-    factory = kwargs.get("factory", Connection)
-    return factory(database, **kwargs)
+def connect(database, timeout=5.0, detect_types=0, isolation_level="",
+                 check_same_thread=True, factory=None, cached_statements=100):
+    factory = Connection if not factory else factory
+    return factory(database, timeout, detect_types, isolation_level,
+                    check_same_thread, factory, cached_statements)
 
 
 def _unicode_text_factory(x):
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -28,9 +28,11 @@
         return result
 
     elif value.typePtr == typeCache.BooleanType:
-        return result
+        return bool(value.internalRep.longValue)
     elif value.typePtr == typeCache.ByteArrayType:
-        return result
+        size = tkffi.new('int*')
+        data = tklib.Tcl_GetByteArrayFromObj(value, size)
+        return tkffi.buffer(data, size[0])[:]
     elif value.typePtr == typeCache.DoubleType:
         return value.internalRep.doubleValue
     elif value.typePtr == typeCache.IntType:
@@ -50,7 +52,7 @@
             result.append(FromObj(app, tcl_elem[0]))
         return tuple(result)
     elif value.typePtr == typeCache.ProcBodyType:
-        return result
+        pass  # fall through and return tcl object.
     elif value.typePtr == typeCache.StringType:
         buf = tklib.Tcl_GetUnicode(value)
         length = tklib.Tcl_GetCharLength(value)
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -72,6 +72,7 @@
 int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
 char *Tcl_GetString(Tcl_Obj* objPtr);
 char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+unsigned char *Tcl_GetByteArrayFromObj(Tcl_Obj* objPtr, int* lengthPtr);
 
 Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
 int Tcl_GetCharLength(Tcl_Obj* objPtr);
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -10,8 +10,35 @@
 
 import os
 def get_include():
-    head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.join(head, '../include')
+    """
+    Return the directory that contains the NumPy \\*.h header files.
+
+    Extension modules that need to compile against NumPy should use this
+    function to locate the appropriate include directory.
+
+    Notes
+    -----
+    When using ``distutils``, for example in ``setup.py``.
+    ::
+
+        import numpy as np
+        ...
+        Extension('extension_name', ...
+                include_dirs=[np.get_include()])
+        ...
+
+    """
+    import numpy
+    if getattr(numpy, 'show_config', None) is None:
+        # running from numpy source directory
+        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
+        return os.path.join(head, '../include')
+    else:
+        # using installed numpy core headers
+        import numpy.core as core
+        d = os.path.join(os.path.dirname(core.__file__), 'include')
+    return d
+
 
 
 __all__ = ['__version__', 'get_include']
diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -1,1 +1,75 @@
 from _numpypy.numerictypes import *
+import numpypy
+
+def issubclass_(arg1, arg2):
+    """
+    Determine if a class is a subclass of a second class.
+
+    `issubclass_` is equivalent to the Python built-in ``issubclass``,
+    except that it returns False instead of raising a TypeError is one
+    of the arguments is not a class.
+
+    Parameters
+    ----------
+    arg1 : class
+        Input class. True is returned if `arg1` is a subclass of `arg2`.
+    arg2 : class or tuple of classes.
+        Input class. If a tuple of classes, True is returned if `arg1` is a
+        subclass of any of the tuple elements.
+
+    Returns
+    -------
+    out : bool
+        Whether `arg1` is a subclass of `arg2` or not.
+
+    See Also
+    --------
+    issubsctype, issubdtype, issctype
+
+    Examples
+    --------
+    >>> np.issubclass_(np.int32, np.int)
+    True
+    >>> np.issubclass_(np.int32, np.float)
+    False
+
+    """
+    try:
+        return issubclass(arg1, arg2)
+    except TypeError:
+        return False
+
+def issubdtype(arg1, arg2):
+    """
+    Returns True if first argument is a typecode lower/equal in type hierarchy.
+
+    Parameters
+    ----------
+    arg1, arg2 : dtype_like
+        dtype or string representing a typecode.
+
+    Returns
+    -------
+    out : bool
+
+    See Also
+    --------
+    issubsctype, issubclass_
+    numpy.core.numerictypes : Overview of numpy type hierarchy.
+
+    Examples
+    --------
+    >>> np.issubdtype('S1', str)
+    True
+    >>> np.issubdtype(np.float64, np.float32)
+    False
+
+    """
+    if issubclass_(arg2, generic):
+        return issubclass(numpypy.dtype(arg1).type, arg2)
+    mro = numpypy.dtype(arg2).type.mro()
+    if len(mro) > 1:
+        val = mro[1]
+    else:
+        val = mro[0]
+    return issubclass(numpypy.dtype(arg1).type, val)
diff --git a/pypy/TODO b/pypy/TODO
deleted file mode 100644
--- a/pypy/TODO
+++ /dev/null
@@ -1,2 +0,0 @@
-
-* ARM
diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst
--- a/pypy/doc/arm.rst
+++ b/pypy/doc/arm.rst
@@ -35,6 +35,11 @@
   * ``qemu-system``
   * ``qemu-user-static``
 
+- The dependencies above are in addition to the ones needed for a regular
+  translation, `listed here`_.
+
+.. _`listed here`: getting-started-python.html#translating-the-pypy-python-interpreter
+
 
 Creating a Qemu based ARM chroot
 --------------------------------
diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -107,9 +107,15 @@
         for i in range(min(len(varnames), self.getfastscopelength())):
             name = varnames[i]
             w_value = fastscope_w[i]
+            w_name = self.space.wrap(name)
             if w_value is not None:
-                w_name = self.space.wrap(name)
                 self.space.setitem(self.w_locals, w_name, w_value)
+            else:
+                try:
+                    self.space.delitem(self.w_locals, w_name)
+                except OperationError as e:
+                    if not e.match(self.space, self.space.w_KeyError):
+                        raise
 
     def locals2fast(self):
         # Copy values from self.w_locals to the fastlocals
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -954,6 +954,8 @@
 
         sys.path.append(self.goal_dir)
         # make sure cwd does not contain a stdlib
+        if self.tmp_dir.startswith(self.trunkdir):
+            skip('TMPDIR is inside the PyPy source')
         os.chdir(self.tmp_dir)
         tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c')
         try:
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -83,10 +83,21 @@
     def test_locals(self):
         def f():
             return locals()
+
         def g(c=0, b=0, a=0):
             return locals()
+
         assert f() == {}
-        assert g() == {'a':0, 'b':0, 'c':0}
+        assert g() == {'a': 0, 'b': 0, 'c': 0}
+
+    def test_locals_deleted_local(self):
+        def f():
+            a = 3
+            locals()
+            del a
+            return locals()
+
+        assert f() == {}
 
     def test_dir(self):
         def f():
@@ -252,25 +263,9 @@
         assert next(x) == 3
 
     def test_xrange_args(self):
-##        # xrange() attributes are deprecated and were removed in Python 2.3.
-##        x = xrange(2)
-##        assert x.start == 0
-##        assert x.stop == 2
-##        assert x.step == 1
-
-##        x = xrange(2,10,2)
-##        assert x.start == 2
-##        assert x.stop == 10
-##        assert x.step == 2
-
-##        x = xrange(2.3, 10.5, 2.4)
-##        assert x.start == 2
-##        assert x.stop == 10
-##        assert x.step == 2
-
         raises(ValueError, xrange, 0, 1, 0)
 
-    def test_xrange_repr(self): 
+    def test_xrange_repr(self):
         assert repr(xrange(1)) == 'xrange(1)'
         assert repr(xrange(1,2)) == 'xrange(1, 2)'
         assert repr(xrange(1,2,3)) == 'xrange(1, 4, 3)'
@@ -329,7 +324,7 @@
         raises(TypeError, xrange, 1, 3+2j)
         raises(TypeError, xrange, 1, 2, '1')
         raises(TypeError, xrange, 1, 2, 3+2j)
-    
+
     def test_sorted(self):
         l = []
         sorted_l = sorted(l)
@@ -348,7 +343,7 @@
         assert sorted_l is not l
         assert sorted_l == ['C', 'b', 'a']
         raises(TypeError, sorted, [], reverse=None)
-        
+
     def test_reversed_simple_sequences(self):
         l = range(5)
         rev = reversed(l)
@@ -364,8 +359,8 @@
                 return 42
         obj = SomeClass()
         assert reversed(obj) == 42
-    
-        
+
+
     def test_cmp(self):
         assert cmp(9,9) == 0
         assert cmp(0,9) < 0
@@ -398,7 +393,7 @@
         raises(RuntimeError, cmp, a, c)
         # okay, now break the cycles
         a.pop(); b.pop(); c.pop()
-        
+
     def test_coerce(self):
         assert coerce(1, 2)    == (1, 2)
         assert coerce(1L, 2L)  == (1L, 2L)
@@ -465,7 +460,7 @@
         assert eval("1+2") == 3
         assert eval(" \t1+2\n") == 3
         assert eval("len([])") == 0
-        assert eval("len([])", {}) == 0        
+        assert eval("len([])", {}) == 0
         # cpython 2.4 allows this (raises in 2.3)
         assert eval("3", None, None) == 3
         i = 4
@@ -683,15 +678,15 @@
         w_value = space.getitem(w_dict, space.wrap('i'))
         assert space.eq_w(w_value, space.wrap(42))
 
-    def test_execfile_different_lineendings(self, space): 
+    def test_execfile_different_lineendings(self, space):
         from rpython.tool.udir import udir
         d = udir.ensure('lineending', dir=1)
-        dos = d.join('dos.py') 
-        f = dos.open('wb') 
+        dos = d.join('dos.py')
+        f = dos.open('wb')
         f.write("x=3\r\n\r\ny=4\r\n")
-        f.close() 
+        f.close()
         space.appexec([space.wrap(str(dos))], """
-            (filename): 
+            (filename):
                 d = {}
                 execfile(filename, d)
                 assert d['x'] == 3
@@ -699,12 +694,12 @@
         """)
 
         unix = d.join('unix.py')
-        f = unix.open('wb') 
+        f = unix.open('wb')
         f.write("x=5\n\ny=6\n")
-        f.close() 
+        f.close()
 
         space.appexec([space.wrap(str(unix))], """
-            (filename): 
+            (filename):
                 d = {}
                 execfile(filename, d)
                 assert d['x'] == 5
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -400,6 +400,8 @@
     def test_socket_connect_ex(self):
         import _socket
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+        # The following might fail if the DNS redirects failed requests to a
+        # catch-all address (i.e. opendns).
         # Make sure we get an app-level error, not an interp one.
         raises(_socket.gaierror, s.connect_ex, ("wrong.invalid", 80))
         s.close()
diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -149,14 +149,17 @@
          only used if the array is constructed that way. Almost always this
          parameter is NULL.
     """
-    if min_depth !=0 or max_depth != 0:
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            '_PyArray_FromAny called with not-implemented min_dpeth or max_depth argument'))
     if requirements not in (0, NPY_DEFAULT):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             '_PyArray_FromAny called with not-implemented requirements argument'))
     w_array = array(space, w_obj, w_dtype=w_dtype, copy=False)
-    if w_array.is_scalar():
+    if min_depth !=0 and len(w_array.get_shape()) < min_depth:
+        raise OperationError(space.w_ValueError, space.wrap(
+            'object of too small depth for desired array'))
+    elif max_depth !=0 and len(w_array.get_shape()) > max_depth:
+        raise OperationError(space.w_ValueError, space.wrap(
+            'object of too deep for desired array'))
+    elif w_array.is_scalar():
         # since PyArray_DATA() fails on scalars, create a 1D array and set empty
         # shape. So the following combination works for *reading* scalars:
         #     PyObject *arr = PyArray_FromAny(obj);
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -103,11 +103,13 @@
     exc_p[0] = make_ref(space, operr.w_type)
     val_p[0] = make_ref(space, operr.get_w_value(space))
 
- at cpython_api([], lltype.Void)
+ at cpython_api([], rffi.INT_real, error=0)
 def PyErr_BadArgument(space):
     """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where
     message indicates that a built-in operation was invoked with an illegal
-    argument.  It is mostly for internal use."""
+    argument.  It is mostly for internal use. In CPython this function always
+    raises an exception and returns 0 in all cases, hence the (ab)use of the
+    error indicator."""
     raise OperationError(space.w_TypeError,
             space.wrap("bad argument type for built-in operation"))
 
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -90,15 +90,16 @@
     def test_FromAny(self, space, api):
         a = array(space, [10, 5, 3])
         assert api._PyArray_FromAny(a, NULL, 0, 0, 0, NULL) is a
-        self.raises(space, api, NotImplementedError, api._PyArray_FromAny,
-                    a, NULL, 0, 3, 0, NULL)
+        assert api._PyArray_FromAny(a, NULL, 1, 4, 0, NULL) is a
+        self.raises(space, api, ValueError, api._PyArray_FromAny,
+                    a, NULL, 4, 5, 0, NULL)
 
     def test_FromObject(self, space, api):
         a = array(space, [10, 5, 3])
         assert api._PyArray_FromObject(a, a.get_dtype().num, 0, 0) is a
-        exc = self.raises(space, api, NotImplementedError, api._PyArray_FromObject,
-                    a, 11, 0, 3)
-        assert exc.errorstr(space).find('FromObject') >= 0
+        exc = self.raises(space, api, ValueError, api._PyArray_FromObject,
+                    a, 11, 4, 5)
+        assert exc.errorstr(space).find('desired') >= 0
 
     def test_list_from_fixedptr(self, space, api):
         A = lltype.GcArray(lltype.Float)
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -70,9 +70,10 @@
         api.PyErr_Clear()
 
     def test_BadArgument(self, space, api):
-        api.PyErr_BadArgument()
+        ret = api.PyErr_BadArgument()
         state = space.fromcache(State)
         assert space.eq_w(state.operror.w_type, space.w_TypeError)
+        assert ret == 0
         api.PyErr_Clear()
 
     def test_Warning(self, space, api, capfd):
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -91,6 +91,7 @@
         invalid = rffi.str2charp('invalid')
         utf_8 = rffi.str2charp('utf-8')
         prev_encoding = rffi.str2charp(space.unwrap(w_default_encoding))
+        self.raises(space, api, TypeError, api.PyUnicode_SetDefaultEncoding, lltype.nullptr(rffi.CCHARP.TO))
         assert api.PyUnicode_SetDefaultEncoding(invalid) == -1
         assert api.PyErr_Occurred() is space.w_LookupError
         api.PyErr_Clear()
@@ -316,6 +317,15 @@
         rffi.free_charp(b_text)
         rffi.free_charp(b_encoding)
 
+    def test_decode_null_encoding(self, space, api):
+        null_charp = lltype.nullptr(rffi.CCHARP.TO)
+        u_text = u'abcdefg'
+        s_text = space.str_w(api.PyUnicode_AsEncodedString(space.wrap(u_text), null_charp, null_charp))
+        b_text = rffi.str2charp(s_text)
+        assert space.unwrap(api.PyUnicode_Decode(b_text, len(s_text), null_charp, null_charp)) == u_text
+        self.raises(space, api, TypeError, api.PyUnicode_FromEncodedObject, space.wrap(u_text), null_charp, None)
+        rffi.free_charp(b_text)
+
     def test_leak(self):
         size = 50
         raw_buf, gc_buf = rffi.alloc_buffer(size)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -273,6 +273,8 @@
 def PyUnicode_SetDefaultEncoding(space, encoding):
     """Sets the currently active default encoding. Returns 0 on
     success, -1 in case of an error."""
+    if not encoding:
+        PyErr_BadArgument(space)
     w_encoding = space.wrap(rffi.charp2str(encoding))
     setdefaultencoding(space, w_encoding)
     default_encoding[0] = '\x00'
@@ -350,8 +352,11 @@
     in the unicode() built-in function.  The codec to be used is looked up
     using the Python codec registry.  Return NULL if an exception was raised by
     the codec."""
+    if not encoding:
+        # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead
+        encoding = PyUnicode_GetDefaultEncoding(space)
+    w_encoding = space.wrap(rffi.charp2str(encoding))
     w_str = space.wrap(rffi.charpsize2str(s, size))
-    w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))
     else:
@@ -379,6 +384,9 @@
 
     All other objects, including Unicode objects, cause a TypeError to be
     set."""
+    if not encoding:
+        raise OperationError(space.w_TypeError,
+                             space.wrap("decoding Unicode is not supported"))
     w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -334,9 +334,8 @@
             v = hi
         del partials[added:]
         if v != 0.0:
-            if rfloat.isinf(v) or rfloat.isnan(v):
-                if (not rfloat.isinf(original) and
-                    not rfloat.isnan(original)):
+            if not rfloat.isfinite(v):
+                if rfloat.isfinite(original):
                     raise OperationError(space.w_OverflowError,
                                          space.wrap("intermediate overflow"))
                 if rfloat.isinf(original):
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -319,6 +319,15 @@
         else:
             self.done_func = None
 
+    def are_common_types(self, dtype1, dtype2):
+        if dtype1.is_complex_type() and dtype2.is_complex_type():
+            return True
+        elif not (dtype1.is_complex_type() or dtype2.is_complex_type()) and \
+                (dtype1.is_int_type() and dtype2.is_int_type() or dtype1.is_float_type() and dtype2.is_float_type()) and \
+                not (dtype1.is_bool_type() or dtype2.is_bool_type()):
+            return True
+        return False
+
     @jit.unroll_safe
     def call(self, space, args_w):
         if len(args_w) > 2:
@@ -339,6 +348,12 @@
                  'unsupported operand dtypes %s and %s for "%s"' % \
                  (w_rdtype.get_name(), w_ldtype.get_name(),
                   self.name)))
+
+        if self.are_common_types(w_ldtype, w_rdtype):
+            if not w_lhs.is_scalar() and w_rhs.is_scalar():
+                w_rdtype = w_ldtype
+            elif w_lhs.is_scalar() and not w_rhs.is_scalar():
+                w_ldtype = w_rdtype
         calc_dtype = find_binop_result_dtype(space,
             w_ldtype, w_rdtype,
             int_only=self.int_only,
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2965,6 +2965,17 @@
 
         assert len(list(a[0])) == 2
 
+    def test_issue_1589(self):
+        import numpypy as numpy
+        c = numpy.array([[(1, 2, 'a'), (3, 4, 'b')], [(5, 6, 'c'), (7, 8, 'd')]],
+                        dtype=[('bg', 'i8'), ('fg', 'i8'), ('char', 'S1')])
+        assert c[0][0]["char"] == 'a'
+
+    def test_scalar_coercion(self):
+        import numpypy as np
+        a = np.array([1,2,3], dtype=np.int16)
+        assert (a * 2).dtype == np.int16
+
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1766,14 +1766,14 @@
 
     def store(self, arr, i, offset, box):
         assert isinstance(box, interp_boxes.W_StringBox)
-        # XXX simplify to range(box.dtype.get_size()) ?
         return self._store(arr.storage, i, offset, box)
 
     @jit.unroll_safe
     def _store(self, storage, i, offset, box):
         assert isinstance(box, interp_boxes.W_StringBox)
-        for k in range(min(self.size, box.arr.size-offset)):
-            storage[k + i] = box.arr.storage[k + offset]
+        # XXX simplify to range(box.dtype.get_size()) ?
+        for k in range(min(self.size, box.arr.size-box.ofs)):
+            storage[k + offset + i] = box.arr.storage[k + box.ofs]
 
     def read(self, arr, i, offset, dtype=None):
         if dtype is None:
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -149,6 +149,8 @@
         interpleveldefs['nice'] = 'interp_posix.nice'
     if hasattr(os, 'getlogin'):
         interpleveldefs['getlogin'] = 'interp_posix.getlogin'
+    if hasattr(os, 'ctermid'):
+        interpleveldefs['ctermid'] = 'interp_posix.ctermid'
 
     for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid',
                  'seteuid', 'setgid', 'setegid', 'getgroups', 'getpgrp',
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1218,3 +1218,10 @@
         return space.wrap(rurandom.urandom(context, n))
     except OSError, e:
         raise wrap_oserror(space, e)
+
+def ctermid(space):
+    """ctermid() -> string
+
+    Return the name of the controlling terminal for this process.
+    """
+    return space.wrap(os.ctermid())
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -110,7 +110,7 @@
                        '__pypy__', 'cStringIO', '_collections', 'struct',
                        'mmap', 'marshal', '_codecs', 'rctime', 'cppyy',
                        '_cffi_backend', 'pyexpat', '_continuation', '_io',
-                       'thread']:
+                       'thread', 'select']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -49,12 +49,15 @@
     from pypy.module.thread.os_lock import Lock
     assert pypypolicy.look_inside_function(Lock.descr_lock_acquire.im_func)
 
+def test_select():
+    from pypy.module.select.interp_select import poll
+    assert pypypolicy.look_inside_function(poll)
+
 def test_pypy_module():
     from pypy.module._collections.interp_deque import W_Deque
     from pypy.module._random.interp_random import W_Random
     assert not pypypolicy.look_inside_function(W_Random.random)
     assert pypypolicy.look_inside_function(W_Deque.length)
-    assert not pypypolicy.look_inside_pypy_module('select.interp_epoll')
     assert pypypolicy.look_inside_pypy_module('__builtin__.operation')
     assert pypypolicy.look_inside_pypy_module('__builtin__.abstractinst')
     assert pypypolicy.look_inside_pypy_module('__builtin__.functional')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -277,3 +277,28 @@
             f1 = call_release_gil(..., descr=<Calli 4 ii EF=6 OS=62>)
             ...
         """)
+
+    def test__cffi_bug1(self):
+        from rpython.rlib.test.test_clibffi import get_libm_name
+        def main(libm_name):
+            try:
+                import _cffi_backend
+            except ImportError:
+                sys.stderr.write('SKIP: cannot import _cffi_backend\n')
+                return 0
+
+            libm = _cffi_backend.load_library(libm_name)
+            BDouble = _cffi_backend.new_primitive_type("double")
+            BSin = _cffi_backend.new_function_type([BDouble], BDouble)
+            sin = libm.load_function(BSin, 'sin')
+
+            def f(*args):
+                for i in range(300):
+                    sin(*args)
+
+            f(1.0)
+            f(1)
+        #
+        libm_name = get_libm_name(sys.platform)
+        log = self.run(main, [libm_name])
+        # assert did not crash
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -113,7 +113,7 @@
             i13 = strgetitem(p9, 0)
             i15 = int_eq(i13, 45)
             guard_false(i15, descr=...)
-            i17 = int_sub(0, i10)
+            i17 = int_neg(i10)
             i19 = int_gt(i10, 23)
             guard_false(i19, descr=...)
             p21 = newstr(23)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -1,4 +1,3 @@
-import py
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
 
@@ -51,7 +50,6 @@
         """)
 
     def test_lock_acquire_release(self):
-        py.test.skip("test too precise, please fix me")
         def main(n):
             import threading
             lock = threading.Lock()
@@ -62,35 +60,30 @@
         assert log.result == main(500)
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-        i58 = int_gt(i43, 0)
-        guard_true(i58, descr=<Guard0x10483adb8>)
-        p59 = getfield_gc(p15, descr=<FieldP pypy.module.thread.os_lock.Lock.inst_lock 8>)
-        i60 = getfield_gc(p59, descr=<FieldU rpython.rlib.rthread.Lock.inst__lock 8>)
+        i55 = int_gt(i43, 0)
+        guard_true(i55, descr=...)
+        p56 = force_token()
+        setfield_gc(p0, p56, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        i57 = call_release_gil(..., i36, 1, descr=<Calli 4 ii EF=6>)
+        guard_not_forced(descr=...)
+        guard_no_exception(descr=...)
+        i58 = int_is_true(i57)
+        guard_true(i58, descr=...)
+        i59 = int_sub(i43, 1)
+        guard_not_invalidated(descr=...)
         p61 = force_token()
-        setfield_gc(p0, p61, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 24>)
-        i62 = call_release_gil(4312440032, i60, 1, descr=<Calli 4 ii EF=6>)
-        guard_not_forced(descr=<Guard0x103f3cca0>)
-        guard_no_exception(descr=<Guard0x10483ad40>)
+        setfield_gc(p0, p61, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        i62 = call_release_gil(..., i36, 0, descr=<Calli 4 ii EF=6>)
+        guard_not_forced(descr=...)
+        guard_no_exception(descr=...)
         i63 = int_is_true(i62)
-        guard_true(i63, descr=<Guard0x10483acc8>)
-        i64 = int_sub(i43, 1)
-        guard_not_invalidated(descr=<Guard0x10483ac50>)
-        p66 = getfield_gc(p15, descr=<FieldP pypy.module.thread.os_lock.Lock.inst_lock 8>)
-        i67 = getfield_gc(p66, descr=<FieldU rpython.rlib.rthread.Lock.inst__lock 8>)
-        p68 = force_token()
-        setfield_gc(p0, p68, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 24>)
-        i69 = call_release_gil(4312440032, i67, 0, descr=<Calli 4 ii EF=6>)
-        guard_not_forced(descr=<Guard0x103f3cc20>)
-        guard_no_exception(descr=<Guard0x10483aae8>)
-        i70 = int_is_true(i69)
-        guard_false(i70, descr=<Guard0x10483aa70>)
-        i71 = getfield_gc(p66, descr=<FieldU rpython.rlib.rthread.Lock.inst__lock 8>)
-        p72 = force_token()
-        setfield_gc(p0, p72, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 24>)
-        call_release_gil(4312441056, i71, descr=<Callv 0 i EF=6>)
-        guard_not_forced(descr=<Guard0x103f3cba0>)
-        guard_no_exception(descr=<Guard0x10483a9f8>)
-        guard_not_invalidated(descr=<Guard0x10483a980>)
+        guard_false(i63, descr=...)
+        p64 = force_token()
+        setfield_gc(p0, p64, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        call_release_gil(..., i36, descr=<Callv 0 i EF=6>)
+        guard_not_forced(descr=...)
+        guard_no_exception(descr=...)
+        guard_not_invalidated(descr=...)
         --TICK--
-        jump(..., descr=TargetToken(4361239720))
+        jump(..., descr=...)
         """)
diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -180,12 +180,12 @@
                         i = 0
                         for w_ev in space.listview(w_changelist):
                             ev = space.interp_w(W_Kevent, w_ev)
-                            changelist[i].c_ident = ev.event.c_ident
-                            changelist[i].c_filter = ev.event.c_filter
-                            changelist[i].c_flags = ev.event.c_flags
-                            changelist[i].c_fflags = ev.event.c_fflags
-                            changelist[i].c_data = ev.event.c_data
-                            changelist[i].c_udata = ev.event.c_udata
+                            changelist[i].c_ident = ev.ident
+                            changelist[i].c_filter = ev.filter
+                            changelist[i].c_flags = ev.flags
+                            changelist[i].c_fflags = ev.fflags
+                            changelist[i].c_data = ev.data
+                            changelist[i].c_udata = ev.udata
                             i += 1
                         pchangelist = changelist
                     else:
@@ -206,13 +206,12 @@
                             evt = eventlist[i]
 
                             w_event = W_Kevent(space)
-                            w_event.event = lltype.malloc(kevent, flavor="raw")
-                            w_event.event.c_ident = evt.c_ident
-                            w_event.event.c_filter = evt.c_filter
-                            w_event.event.c_flags = evt.c_flags
-                            w_event.event.c_fflags = evt.c_fflags
-                            w_event.event.c_data = evt.c_data
-                            w_event.event.c_udata = evt.c_udata
+                            w_event.ident = evt.c_ident
+                            w_event.filter = evt.c_filter
+                            w_event.flags = evt.c_flags
+                            w_event.fflags = evt.c_fflags
+                            w_event.data = evt.c_data
+                            w_event.udata = evt.c_udata
 
                             elist_w[i] = w_event
 
@@ -234,11 +233,12 @@
 
 class W_Kevent(W_Root):
     def __init__(self, space):
-        self.event = lltype.nullptr(kevent)
-
-    def __del__(self):
-        if self.event:
-            lltype.free(self.event, flavor="raw")
+        self.ident = rffi.cast(kevent.c_ident, 0)
+        self.filter = rffi.cast(kevent.c_filter, 0)
+        self.flags = rffi.cast(kevent.c_flags, 0)
+        self.fflags = rffi.cast(kevent.c_fflags, 0)
+        self.data = rffi.cast(kevent.c_data, 0)
+        self.udata = lltype.nullptr(rffi.VOIDP.TO)
 
     @unwrap_spec(filter=int, flags='c_uint', fflags='c_uint', data=int, udata=r_uint)
     def descr__init__(self, space, w_ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=r_uint(0)):
@@ -247,35 +247,34 @@
         else:
             ident = r_uint(space.c_filedescriptor_w(w_ident))
 
-        self.event = lltype.malloc(kevent, flavor="raw")
-        rffi.setintfield(self.event, "c_ident", ident)
-        rffi.setintfield(self.event, "c_filter", filter)
-        rffi.setintfield(self.event, "c_flags", flags)
-        rffi.setintfield(self.event, "c_fflags", fflags)
-        rffi.setintfield(self.event, "c_data", data)
-        self.event.c_udata = rffi.cast(rffi.VOIDP, udata)
+        self.ident = rffi.cast(kevent.c_ident, ident)
+        self.filter = rffi.cast(kevent.c_filter, filter)
+        self.flags = rffi.cast(kevent.c_flags, flags)
+        self.fflags = rffi.cast(kevent.c_fflags, fflags)
+        self.data = rffi.cast(kevent.c_data, data)
+        self.udata = rffi.cast(rffi.VOIDP, udata)
 
     def _compare_all_fields(self, other, op):
         if IDENT_UINT:
-            l_ident = rffi.cast(lltype.Unsigned, self.event.c_ident)
-            r_ident = rffi.cast(lltype.Unsigned, other.event.c_ident)
+            l_ident = rffi.cast(lltype.Unsigned, self.ident)
+            r_ident = rffi.cast(lltype.Unsigned, other.ident)
         else:
-            l_ident = self.event.c_ident
-            r_ident = other.event.c_ident
-        l_filter = rffi.cast(lltype.Signed, self.event.c_filter)
-        r_filter = rffi.cast(lltype.Signed, other.event.c_filter)
-        l_flags = rffi.cast(lltype.Unsigned, self.event.c_flags)
-        r_flags = rffi.cast(lltype.Unsigned, other.event.c_flags)
-        l_fflags = rffi.cast(lltype.Unsigned, self.event.c_fflags)
-        r_fflags = rffi.cast(lltype.Unsigned, other.event.c_fflags)
+            l_ident = self.ident
+            r_ident = other.ident
+        l_filter = rffi.cast(lltype.Signed, self.filter)
+        r_filter = rffi.cast(lltype.Signed, other.filter)
+        l_flags = rffi.cast(lltype.Unsigned, self.flags)
+        r_flags = rffi.cast(lltype.Unsigned, other.flags)
+        l_fflags = rffi.cast(lltype.Unsigned, self.fflags)
+        r_fflags = rffi.cast(lltype.Unsigned, other.fflags)
         if IDENT_UINT:
-            l_data = rffi.cast(lltype.Signed, self.event.c_data)
-            r_data = rffi.cast(lltype.Signed, other.event.c_data)
+            l_data = rffi.cast(lltype.Signed, self.data)
+            r_data = rffi.cast(lltype.Signed, other.data)
         else:
-            l_data = self.event.c_data
-            r_data = other.event.c_data
-        l_udata = rffi.cast(lltype.Unsigned, self.event.c_udata)
-        r_udata = rffi.cast(lltype.Unsigned, other.event.c_udata)
+            l_data = self.data
+            r_data = other.data
+        l_udata = rffi.cast(lltype.Unsigned, self.udata)
+        r_udata = rffi.cast(lltype.Unsigned, other.udata)
 
         if op == "eq":
             return l_ident == r_ident and \
@@ -330,22 +329,22 @@
         return space.wrap(self.compare_all_fields(space, w_other, "gt"))
 
     def descr_get_ident(self, space):
-        return space.wrap(self.event.c_ident)
+        return space.wrap(self.ident)
 
     def descr_get_filter(self, space):
-        return space.wrap(self.event.c_filter)
+        return space.wrap(self.filter)
 
     def descr_get_flags(self, space):
-        return space.wrap(self.event.c_flags)
+        return space.wrap(self.flags)
 
     def descr_get_fflags(self, space):
-        return space.wrap(self.event.c_fflags)
+        return space.wrap(self.fflags)
 
     def descr_get_data(self, space):
-        return space.wrap(self.event.c_data)
+        return space.wrap(self.data)
 
     def descr_get_udata(self, space):
-        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.event.c_udata))
+        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.udata))
 
 
 W_Kevent.typedef = TypeDef("select.kevent",
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -31,6 +31,12 @@
     result = list(cursor)
     assert result == [(42,)]
 
+def test_connect_takes_same_positional_args_as_Connection(con):
+    from inspect import getargspec
+    clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
+    conargs = getargspec(_sqlite3.connect).args
+    assert clsargs == conargs
+
 def test_total_changes_after_close(con):
     con.close()
     pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -26,6 +26,8 @@
 class Lock(W_Root):
     "A box around an interp-level lock object."
 
+    _immutable_fields_ = ["lock"]
+
     def __init__(self, space):
         self.space = space
         try:
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -1,5 +1,5 @@
 import operator
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.objspace.std import model, newformat
 from pypy.objspace.std.floattype import float_typedef, W_AbstractFloatObject
 from pypy.objspace.std.multimethod import FailedToImplementArgs
@@ -424,21 +424,33 @@
     x = w_float1.floatval
     y = w_float2.floatval
 
+    try:
+        result = _pow(space, x, y)
+    except PowDomainError:
+        raise operationerrfmt(space.w_ValueError,
+                              "negative number cannot be raised to a "
+                              "fractional power")
+    return W_FloatObject(result)
+
+class PowDomainError(ValueError):
+    """Signals a negative number raised to a fractional power"""
+
+def _pow(space, x, y):
     # Sort out special cases here instead of relying on pow()
-    if y == 2.0:                      # special case for performance:
-        return W_FloatObject(x * x)   # x * x is always correct
+    if y == 2.0:       # special case for performance:
+        return x * x   # x * x is always correct
     if y == 0.0:
         # x**0 is 1, even 0**0
-        return W_FloatObject(1.0)
+        return 1.0
     if isnan(x):
         # nan**y = nan, unless y == 0
-        return W_FloatObject(x)
+        return x
     if isnan(y):
         # x**nan = nan, unless x == 1; x**nan = x
         if x == 1.0:
-            return W_FloatObject(1.0)
+            return 1.0
         else:
-            return W_FloatObject(y)
+            return y
     if isinf(y):
         # x**inf is: 0.0 if abs(x) < 1; 1.0 if abs(x) == 1; inf if
         # abs(x) > 1 (including case where x infinite)
@@ -447,11 +459,11 @@
         # abs(x) > 1 (including case where v infinite)
         x = abs(x)
         if x == 1.0:
-            return W_FloatObject(1.0)
+            return 1.0
         elif (y > 0.0) == (x > 1.0):
-            return W_FloatObject(INFINITY)
+            return INFINITY
         else:
-            return W_FloatObject(0.0)
+            return 0.0
     if isinf(x):
         # (+-inf)**w is: inf for w positive, 0 for w negative; in oth
         # cases, we need to add the appropriate sign if w is an odd
@@ -459,14 +471,14 @@
         y_is_odd = math.fmod(abs(y), 2.0) == 1.0
         if y > 0.0:
             if y_is_odd:
-                return W_FloatObject(x)
+                return x
             else:
-                return W_FloatObject(abs(x))
+                return abs(x)
         else:
             if y_is_odd:
-                return W_FloatObject(copysign(0.0, x))
+                return copysign(0.0, x)
             else:
-                return W_FloatObject(0.0)
+                return 0.0
 
     if x == 0.0:
         if y < 0.0:
@@ -475,16 +487,14 @@
                                             "a negative power"))
 
     negate_result = False
-    # special case: "(-1.0) ** bignum" should not raise ValueError,
+    # special case: "(-1.0) ** bignum" should not raise PowDomainError,
     # unlike "math.pow(-1.0, bignum)".  See http://mail.python.org/
     # -           pipermail/python-bugs-list/2003-March/016795.html
     if x < 0.0:
         if isnan(y):
-            return W_FloatObject(NAN)
+            return NAN
         if math.floor(y) != y:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("negative number cannot be "
-                                            "raised to a fractional power"))
+            raise PowDomainError
         # y is an exact integer, albeit perhaps a very large one.
         # Replace x by its absolute value and remember to negate the
         # pow result if y is odd.
@@ -494,9 +504,9 @@
     if x == 1.0:
         # (-1) ** large_integer also ends up here
         if negate_result:
-            return W_FloatObject(-1.0)
+            return -1.0
         else:
-            return W_FloatObject(1.0)
+            return 1.0
 
     try:
         # We delegate to our implementation of math.pow() the error detection.
@@ -510,7 +520,7 @@
 
     if negate_result:
         z = -z
-    return W_FloatObject(z)
+    return z
 
 
 def neg__Float(space, w_float1):
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -30,7 +30,7 @@
 from rpython.rlib import debug, jit, rerased
 from rpython.rlib.listsort import make_timsort_class
 from rpython.rlib.objectmodel import (
-    instantiate, newlist_hint, resizelist_hint, specialize)
+    instantiate, newlist_hint, resizelist_hint, specialize, import_from_mixin)
 from rpython.tool.sourcetools import func_with_new_name
 
 __all__ = ['W_ListObject', 'make_range_list', 'make_empty_list_with_size']
@@ -1170,7 +1170,6 @@
 
 
 class AbstractUnwrappedStrategy(object):
-    _mixin_ = True
 
     def wrap(self, unwrapped):
         raise NotImplementedError
@@ -1329,7 +1328,6 @@
 
     def setslice(self, w_list, start, step, slicelength, w_other):
         assert slicelength >= 0
-        items = self.unerase(w_list.lstorage)
 
         if self is self.space.fromcache(ObjectListStrategy):
             w_other = w_other._temporarily_as_objects()
@@ -1341,6 +1339,7 @@
             w_list.setslice(start, step, slicelength, w_other_as_object)
             return
 
+        items = self.unerase(w_list.lstorage)
         oldsize = len(items)
         len2 = w_other.length()
         if step == 1:  # Support list resizing for non-extended slices
@@ -1456,7 +1455,9 @@
         self.unerase(w_list.lstorage).reverse()
 
 
-class ObjectListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class ObjectListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "object"
 
@@ -1489,7 +1490,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class IntegerListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class IntegerListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0
     _applevel_repr = "int"
 
@@ -1520,7 +1523,30 @@
         return self.unerase(w_list.lstorage)
 
 
-class FloatListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            l = self.unerase(w_list.lstorage)
+            other = w_other.getitems_int()
+            assert other is not None
+            l += other
+            return
+        return self._base_extend_from_list(w_list, w_other)
+
+
+    _base_setslice = setslice
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            storage = self.erase(w_other.getitems_int())
+            w_other = W_ListObject.from_storage_and_strategy(
+                    self.space, storage, self)
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
+class FloatListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0.0
     _applevel_repr = "float"
 
@@ -1548,7 +1574,9 @@
             l.reverse()
 
 
-class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class StringListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "str"
 
@@ -1579,7 +1607,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class UnicodeListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class UnicodeListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "unicode"
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -225,6 +225,15 @@
         l.setslice(0, 1, 2, W_ListObject(space, [w('a'), w(2), w(3)]))
         assert isinstance(l.strategy, ObjectListStrategy)
 
+    def test_setslice_int_range(self):
+        space = self.space
+        w = space.wrap
+        l = W_ListObject(space, [w(1), w(2), w(3)])
+        assert isinstance(l.strategy, IntegerListStrategy)
+        l.setslice(0, 1, 2, make_range_list(space, 5, 1, 4))
+        assert isinstance(l.strategy, IntegerListStrategy)
+
+
     def test_setslice_List(self):
         space = self.space
 
@@ -467,6 +476,12 @@
         l4 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3), self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         assert self.space.eq_w(l3, l4)
 
+    def test_add_of_range_and_int(self):
+        l1 = make_range_list(self.space, 0, 1, 100)
+        l2 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
+        l3 = self.space.add(l2, l1)
+        assert l3.strategy is l2.strategy
+
     def test_mul(self):
         l1 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         l2 = l1.mul(2)
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -197,10 +197,10 @@
 
 W_AbstractTupleObject.typedef = StdTypeDef(
     "tuple",
-    __doc__ = '''tuple() -> an empty tuple
+    __doc__ = """tuple() -> an empty tuple
 tuple(sequence) -> tuple initialized from sequence's items
 
-If the argument is a tuple, the return value is the same object.''',
+If the argument is a tuple, the return value is the same object.""",
     __new__ = interp2app(W_AbstractTupleObject.descr_new),
     __repr__ = interp2app(W_AbstractTupleObject.descr_repr),
     __hash__ = interpindirect2app(W_AbstractTupleObject.descr_hash),
diff --git a/pypy/pytest-A.py b/pypy/pytest-A.py
--- a/pypy/pytest-A.py
+++ b/pypy/pytest-A.py
@@ -5,7 +5,6 @@
     'arm': ['interpreter/astcompiler/test',
             'interpreter/pyparser/test',
             'interpreter/test',
-            'interpreter/test2',
             'module/test_lib_pypy',
             'objspace/std/test',
     ],
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3442,6 +3442,29 @@
 
         a.build_types(f, [str])
 
+    def test_negative_number_find(self):
+        def f(s, e):
+            return "xyz".find("x", s, e)
+
+        a = self.RPythonAnnotator()
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        def f(s, e):
+            return "xyz".rfind("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+
+        def f(s, e):
+            return "xyz".count("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        
+
     def test_setslice(self):
         def f():
             lst = [2, 5, 7]
@@ -4080,7 +4103,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify instances with no common base class" 
+        assert ("RPython cannot unify instances with no common base class"
                 in exc.value.msg)
 
     def test_unionerror_iters(self):
@@ -4096,7 +4119,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify incompatible iterator variants" in 
+        assert ("RPython cannot unify incompatible iterator variants" in
                 exc.value.msg)
 
     def test_variable_getattr(self):
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -10,7 +10,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeFloat, SomeIterator,
     SomePBC, SomeTypedAddressAccess, SomeAddress, SomeType, s_ImpossibleValue,
     s_Bool, s_None, unionof, missing_operation, add_knowntypedata,
-    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString)
+    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper
 from rpython.annotator import builtin
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
@@ -333,12 +333,13 @@
         check_negative_slice(s_start, s_stop)
         lst.listdef.resize()
 
-def check_negative_slice(s_start, s_stop):
+def check_negative_slice(s_start, s_stop, error="slicing"):
     if isinstance(s_start, SomeInteger) and not s_start.nonneg:
-        raise AnnotatorError("slicing: not proven to have non-negative start")
+        raise AnnotatorError("%s: not proven to have non-negative start" %
+                             error)
     if isinstance(s_stop, SomeInteger) and not s_stop.nonneg and \
            getattr(s_stop, 'const', 0) != -1:
-        raise AnnotatorError("slicing: not proven to have non-negative stop")
+        raise AnnotatorError("%s: not proven to have non-negative stop" % error)
 
 
 class __extend__(SomeDict):
@@ -448,12 +449,15 @@
         return s_Bool
 
     def method_find(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "find")
         return SomeInteger()
 
     def method_rfind(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "rfind")
         return SomeInteger()
 
     def method_count(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "count")
         return SomeInteger(nonneg=True)
 
     def method_strip(str, chr):
@@ -520,6 +524,11 @@
     op_contains.can_only_throw = []
 
 
+class __extend__(SomeByteArray):
+    def getslice(ba, s_start, s_stop):
+        check_negative_slice(s_start, s_stop)
+        return SomeByteArray()
+
 class __extend__(SomeUnicodeString):
     def method_encode(uni, s_enc):
         if not s_enc.is_constant():
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -252,6 +252,23 @@
         from rpython.translator.tool.graphpage import try_show
         try_show(self)
 
+    def get_graph(self):
+        import gc
+        pending = [self]   # pending blocks
+        seen = {self: True, None: True}
+        for x in pending:
+            for y in gc.get_referrers(x):
+                if isinstance(y, FunctionGraph):
+                    return y
+                elif isinstance(y, Link):
+                    block = y.prevblock
+                    if block not in seen:
+                        pending.append(block)
+                        seen[block] = True
+                elif isinstance(y, dict):
+                    pending.append(y)   # go back from the dict to the real obj
+        return pending
+
     view = show
 
 
diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py
--- a/rpython/flowspace/specialcase.py
+++ b/rpython/flowspace/specialcase.py
@@ -1,3 +1,4 @@
+import os
 from rpython.flowspace.model import Constant, const
 
 SPECIAL_CASES = {}
@@ -37,6 +38,18 @@
     return space.frame.do_operation('simple_call', const(isinstance),
             w_instance, w_type)
 
+ at register_flow_sc(open)
+def sc_open(space, *args_w):
+    from rpython.rlib.rfile import create_file
+
+    return space.frame.do_operation("simple_call", const(create_file), *args_w)
+
+ at register_flow_sc(os.tmpfile)
+def sc_os_tmpfile(space):
+    from rpython.rlib.rfile import create_temp_rfile
+
+    return space.frame.do_operation("simple_call", const(create_temp_rfile))
+
 # _________________________________________________________________________
 # a simplified version of the basic printing routines, for RPython programs
 class StdOutBuffer:
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -1063,7 +1063,6 @@
         assert len(graph.startblock.exits) == 1
         assert graph.startblock.exits[0].target == graph.returnblock
 
-
     def test_global_variable(self):
         def global_var_missing():
             return a
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -381,6 +381,8 @@
             res = self.llinterp.eval_graph(ptr._obj.graph, args)
         else:
             res = ptr._obj._callable(*args)
+        if RESULT is lltype.Void:
+            return None
         return support.cast_result(RESULT, res)
 
     def _do_call(self, func, args_i, args_r, args_f, calldescr):
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -87,9 +87,11 @@
     def _escape(self, box):
         if box in self.new_boxes:
             self.new_boxes[box] = False
-        if box in self.dependencies:
-            deps = self.dependencies[box]
-            del self.dependencies[box]
+        try:
+            deps = self.dependencies.pop(box)
+        except KeyError:
+            pass
+        else:
             for dep in deps:
                 self._escape(dep)
 
@@ -117,15 +119,18 @@
             # effects are so well defined.
             elif effectinfo.oopspecindex == effectinfo.OS_ARRAYCOPY:
                 # The destination box
-                if argboxes[2] in self.new_boxes:
-                    # XXX: no descr here so we invalidate any of them, not just
-                    # of the correct type
-                    # XXX: in theory the indices of the copy could be looked at
-                    # as well
-                    for descr, cache in self.heap_array_cache.iteritems():
+                if (
+                    argboxes[2] in self.new_boxes and
+                    len(effectinfo.write_descrs_arrays) == 1
+                ):
+                    # Fish the descr out of the effectinfo
+                    cache = self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
+                    if cache is not None:
+                        # XXX: in theory the indices of the copy could be
+                        # looked at
                         for idx, cache in cache.iteritems():
                             for frombox in cache.keys():
-                                if not self.new_boxes.get(frombox, False):
+                                if not self.is_unescaped(frombox):
                                     del cache[frombox]
                     return
             else:
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -32,7 +32,6 @@
     'please fix rlib/jit.py to say ENABLE_ALL_OPTS = %r' % (ALL_OPTS_NAMES,))
 
 def build_opt_chain(metainterp_sd, enable_opts):
-    config = metainterp_sd.config
     optimizations = []
     unroll = 'unroll' in enable_opts    # 'enable_opts' is normally a dict
     for name, opt in unroll_all_opts:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -107,6 +107,11 @@
         v2 = self.getvalue(op.getarg(1))
         if v2.is_constant() and v2.box.getint() == 0:
             self.make_equal_to(op.result, v1)
+        elif v1.is_constant() and v1.box.getint() == 0:
+            op = op.copy_and_change(rop.INT_NEG, args=[v2.box])
+            self.emit_operation(op)
+        elif v1 is v2:
+            self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
             # Synthesize the reverse ops for optimize_default to reuse
@@ -166,6 +171,8 @@
 
         if v2.is_constant() and v2.box.getint() == 0:
             self.make_equal_to(op.result, v1)
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
 
@@ -175,6 +182,8 @@
 
         if v2.is_constant() and v2.box.getint() == 0:
             self.make_equal_to(op.result, v1)
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -45,7 +45,7 @@
                 return self.optimize_JUMP(op.copy_and_change(rop.JUMP))
             self.last_label_descr = op.getdescr()
         self.emit_operation(op)
-        
+
     def optimize_JUMP(self, op):
         if not self.unroll:
             descr = op.getdescr()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -3735,6 +3735,33 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_sub_identity(self):
+        ops = """
+        [i0]
+        i1 = int_sub(i0, i0)
+        i2 = int_sub(i1, i0)
+        jump(i1, i2)
+        """
+        expected = """
+        [i0]
+        i2 = int_neg(i0)
+        jump(0, i2)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_shift_zero(self):
+        ops = """
+        [i0]
+        i1 = int_lshift(0, i0)
+        i2 = int_rshift(0, i0)
+        jump(i1, i2)
+        """
+        expected = """
+        [i0]
+        jump(0, 0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_bound_and(self):
         ops = """
         [i0]
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -45,6 +45,15 @@
             return value
         return OptValue(self.force_box(optforce))
 
+    def get_args_for_fail(self, modifier):
+        # checks for recursion: it is False unless
+        # we have already seen the very same keybox
+        if self.box is None and not modifier.already_seen_virtual(self.keybox):
+            self._get_args_for_fail(modifier)
+
+    def _get_args_for_fail(self, modifier):
+        raise NotImplementedError("abstract base")
+
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
             return self._make_virtual(modifier)
@@ -193,16 +202,13 @@
             self._cached_sorted_fields = lst
         return lst
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            # checks for recursion: it is False unless
-            # we have already seen the very same keybox
-            lst = self._get_field_descr_list()
-            fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst]
-            modifier.register_virtual_fields(self.keybox, fieldboxes)
-            for ofs in lst:
-                fieldvalue = self._fields[ofs]
-                fieldvalue.get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        lst = self._get_field_descr_list()
+        fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst]
+        modifier.register_virtual_fields(self.keybox, fieldboxes)
+        for ofs in lst:
+            fieldvalue = self._fields[ofs]
+            fieldvalue.get_args_for_fail(modifier)
 
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
@@ -254,18 +260,15 @@
     def set_item_value(self, i, newval):
         raise NotImplementedError
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            # checks for recursion: it is False unless
-            # we have already seen the very same keybox
-            itemboxes = []
-            for i in range(self.getlength()):
-                itemvalue = self.get_item_value(i)
-                itemboxes.append(itemvalue.get_key_box())
-            modifier.register_virtual_fields(self.keybox, itemboxes)
-            for i in range(self.getlength()):
-                itemvalue = self.get_item_value(i)
-                itemvalue.get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        itemboxes = []
+        for i in range(self.getlength()):
+            itemvalue = self.get_item_value(i)
+            itemboxes.append(itemvalue.get_key_box())
+        modifier.register_virtual_fields(self.keybox, itemboxes)
+        for i in range(self.getlength()):
+            itemvalue = self.get_item_value(i)
+            itemvalue.get_args_for_fail(modifier)
 
 
 class VArrayValue(AbstractVArrayValue):
@@ -370,17 +373,16 @@
             descrs.append(item_descrs)
         return descrs
 
-    def get_args_for_fail(self, modifier):
-        if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            itemdescrs = self._get_list_of_descrs()
-            itemboxes = []
-            for i in range(len(self._items)):
-                for descr in itemdescrs[i]:
-                    itemboxes.append(self._items[i][descr].get_key_box())
-            modifier.register_virtual_fields(self.keybox, itemboxes)
-            for i in range(len(self._items)):
-                for descr in itemdescrs[i]:
-                    self._items[i][descr].get_args_for_fail(modifier)
+    def _get_args_for_fail(self, modifier):
+        itemdescrs = self._get_list_of_descrs()
+        itemboxes = []
+        for i in range(len(self._items)):
+            for descr in itemdescrs[i]:
+                itemboxes.append(self._items[i][descr].get_key_box())
+        modifier.register_virtual_fields(self.keybox, itemboxes)
+        for i in range(len(self._items)):
+            for descr in itemdescrs[i]:
+                self._items[i][descr].get_args_for_fail(modifier)
 
     def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
@@ -481,6 +483,15 @@
     def getitem_raw(self, offset, length, descr):
         return self.rawbuffer_value.getitem_raw(self.offset+offset, length, descr)
 
+    def _get_args_for_fail(self, modifier):
+        box = self.rawbuffer_value.get_key_box()
+        modifier.register_virtual_fields(self.keybox, [box])
+        self.rawbuffer_value.get_args_for_fail(modifier)
+
+    def _make_virtual(self, modifier):
+        return modifier.make_vrawslice(self.offset)
+
+
 class OptVirtualize(optimizer.Optimization):
     "Virtualize objects until they escape."
 
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -284,7 +284,10 @@
         return VArrayStructInfo(arraydescr, fielddescrs)
 
     def make_vrawbuffer(self, size, offsets, descrs):
-        return VRawBufferStateInfo(size, offsets, descrs)
+        return VRawBufferInfo(size, offsets, descrs)
+
+    def make_vrawslice(self, offset):
+        return VRawSliceInfo(offset)
 
     def make_vstrplain(self, is_unicode=False):
         if is_unicode:
@@ -554,10 +557,13 @@
             debug_print("\t\t", str(untag(i)))
 
 
-class VRawBufferStateInfo(AbstractVirtualInfo):
+class VAbstractRawInfo(AbstractVirtualInfo):
     kind = INT
     is_about_raw = True
 
+
+class VRawBufferInfo(VAbstractRawInfo):
+
     def __init__(self, size, offsets, descrs):
         self.size = size
         self.offsets = offsets
@@ -580,6 +586,25 @@
             debug_print("\t\t", str(untag(i)))
 
 
+class VRawSliceInfo(VAbstractRawInfo):
+
+    def __init__(self, offset):
+        self.offset = offset
+
+    @specialize.argtype(1)
+    def allocate_int(self, decoder, index):
+        assert len(self.fieldnums) == 1
+        base_buffer = decoder.decode_int(self.fieldnums[0])
+        buffer = decoder.int_add_const(base_buffer, self.offset)
+        decoder.virtuals_cache.set_int(index, buffer)
+        return buffer
+
+    def debug_prints(self):
+        debug_print("\tvrawsliceinfo", " at ",  compute_unique_id(self))
+        for i in self.fieldnums:
+            debug_print("\t\t", str(untag(i)))
+
+
 class VArrayStructInfo(AbstractVirtualInfo):
     def __init__(self, arraydescr, fielddescrs):
         self.arraydescr = arraydescr
@@ -783,7 +808,8 @@
         v = self.virtuals_cache.get_int(index)
         if not v:
             v = self.rd_virtuals[index]
-            assert v.is_about_raw and isinstance(v, VRawBufferStateInfo)
+            ll_assert(bool(v), "resume.py: null rd_virtuals[index]")
+            assert v.is_about_raw and isinstance(v, VAbstractRawInfo)
             v = v.allocate_int(self, index)
             ll_assert(v == self.virtuals_cache.get_int(index), "resume.py: bad cache")
         return v
@@ -1116,6 +1142,10 @@
     def write_a_float(self, index, box):
         self.boxes_f[index] = box
 
+    def int_add_const(self, intbox, offset):
+        return self.metainterp.execute_and_record(rop.INT_ADD, None, intbox,
+                                                  ConstInt(offset))
+
 # ---------- when resuming for blackholing, get direct values ----------
 
 def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage,
@@ -1407,6 +1437,9 @@
     def write_a_float(self, index, float):
         self.blackholeinterp.setarg_f(index, float)
 
+    def int_add_const(self, base, offset):
+        return base + offset
+
 # ____________________________________________________________
 
 def dump_storage(storage, liveboxes):
diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py
--- a/rpython/jit/metainterp/test/test_fficall.py
+++ b/rpython/jit/metainterp/test/test_fficall.py
@@ -86,15 +86,17 @@
             data = rffi.ptradd(exchange_buffer, ofs)
             rffi.cast(lltype.Ptr(TYPE), data)[0] = write_rvalue
 
-        def f():
+        def f(i):
             exbuf = lltype.malloc(rffi.CCHARP.TO, (len(avalues)+2) * 16,
-                                  flavor='raw', zero=True)
-            ofs = 16
+                                  flavor='raw')
+
+            targetptr = rffi.ptradd(exbuf, 16)
             for avalue in unroll_avalues:
                 TYPE = rffi.CArray(lltype.typeOf(avalue))
-                data = rffi.ptradd(exbuf, ofs)
-                rffi.cast(lltype.Ptr(TYPE), data)[0] = avalue
-                ofs += 16
+                if i >= 9:    # a guard that can fail
+                    pass
+                rffi.cast(lltype.Ptr(TYPE), targetptr)[0] = avalue
+                targetptr = rffi.ptradd(targetptr, 16)
 
             jit_ffi_call(cif_description, func_addr, exbuf)
 
@@ -102,8 +104,7 @@
                 res = 654321
             else:
                 TYPE = rffi.CArray(lltype.typeOf(rvalue))
-                data = rffi.ptradd(exbuf, ofs)
-                res = rffi.cast(lltype.Ptr(TYPE), data)[0]
+                res = rffi.cast(lltype.Ptr(TYPE), targetptr)[0]
             lltype.free(exbuf, flavor='raw')
             if lltype.typeOf(res) is lltype.SingleFloat:
                 res = float(res)
@@ -117,9 +118,9 @@
             return res == rvalue
 
         with FakeFFI(fake_call_impl_any):
-            res = f()
+            res = f(-42)
             assert matching_result(res, rvalue)
-            res = self.interp_operations(f, [],
+            res = self.interp_operations(f, [-42],
                             supports_floats = supports_floats,
                           supports_longlong = supports_longlong,
                       supports_singlefloats = supports_singlefloats)
@@ -132,6 +133,19 @@
             self.check_operations_history(call_may_force=0,
                                           call_release_gil=expected_call_release_gil)
 
+            ##################################################
+            driver = jit.JitDriver(reds=['i'], greens=[])
+            def main():
+                i = 0
+                while 1:
+                    driver.jit_merge_point(i=i)
+                    res = f(i)
+                    i += 1
+                    if i == 12:
+                        return res
+            self.meta_interp(main, [])
+
+
     def test_simple_call_int(self):
         self._run([types.signed] * 2, types.signed, [456, 789], -42)
 
diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -29,17 +29,24 @@
 
     OS_ARRAYCOPY = 0
 
-    def __init__(self, extraeffect, oopspecindex):
+    def __init__(self, extraeffect, oopspecindex, write_descrs_arrays):
         self.extraeffect = extraeffect
         self.oopspecindex = oopspecindex
+        self.write_descrs_arrays = write_descrs_arrays
+
 
 class FakeCallDescr(object):
-    def __init__(self, extraeffect, oopspecindex=None):
+    def __init__(self, extraeffect, oopspecindex=None, write_descrs_arrays=[]):
         self.extraeffect = extraeffect
         self.oopspecindex = oopspecindex
+        self.write_descrs_arrays = write_descrs_arrays
 
     def get_extra_info(self):
-        return FakeEffectinfo(self.extraeffect, self.oopspecindex)
+        return FakeEffectinfo(
+            self.extraeffect, self.oopspecindex,
+            write_descrs_arrays=self.write_descrs_arrays
+        )
+
 
 class TestHeapCache(object):
     def test_known_class_box(self):
@@ -364,13 +371,13 @@
         # Just need the destination box for this call
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY),
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
             [None, None, box2, None, None]
         )
         assert h.getarrayitem(box1, index1, descr1) is box2
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY),
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
             [None, None, box3, None, None]
         )
         assert h.getarrayitem(box1, index1, descr1) is None
@@ -379,11 +386,24 @@
         assert h.getarrayitem(box4, index1, descr1) is box2
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY),
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
             [None, None, box2, None, None]
         )
         assert h.getarrayitem(box4, index1, descr1) is None
 
+    def test_ll_arraycopy_differing_descrs(self):
+        h = HeapCache()
+        h.setarrayitem(box1, index1, box2, descr1)
+        assert h.getarrayitem(box1, index1, descr1) is box2
+        h.new_array(box2, lengthbox2)
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr2]),
+            [None, None, box2, None, None]
+        )
+        assert h.getarrayitem(box1, index1, descr1) is box2
+
+
     def test_unescaped(self):
         h = HeapCache()
         assert not h.is_unescaped(box1)
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1201,6 +1201,7 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+            assert self.card_page_indices > 0
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
diff --git a/rpython/memory/support.py b/rpython/memory/support.py
--- a/rpython/memory/support.py
+++ b/rpython/memory/support.py
@@ -121,13 +121,15 @@
                 cur = next
             free_non_gc_object(self)
 
-        def _length_estimate(self):
+        def length(self):
             chunk = self.chunk
+            result = 0
             count = self.used_in_last_chunk
             while chunk:
+                result += count
                 chunk = chunk.next
-                count += chunk_size
-            return count
+                count = chunk_size
+            return result
 
         def foreach(self, callback, arg):
             """Invoke 'callback(address, arg)' for all addresses in the stack.
@@ -144,7 +146,7 @@
         foreach._annspecialcase_ = 'specialize:arg(1)'
 
         def stack2dict(self):
-            result = AddressDict(self._length_estimate())
+            result = AddressDict(self.length())
             self.foreach(_add_in_dict, result)
             return result
 
diff --git a/rpython/memory/test/test_support.py b/rpython/memory/test/test_support.py
--- a/rpython/memory/test/test_support.py
+++ b/rpython/memory/test/test_support.py
@@ -94,6 +94,18 @@
             assert a == addrs[i]
         assert not ll.non_empty()
 
+    def test_length(self):
+        AddressStack = get_address_stack(10)
+        ll = AddressStack()
+        a = raw_malloc(llmemory.sizeof(lltype.Signed))
+        for i in range(42):
+            assert ll.length() == i
+            ll.append(a)
+        for i in range(42-1, -1, -1):
+            b = ll.pop()
+            assert b == a
+            assert ll.length() == i
+
 
 class TestAddressDeque:
     def test_big_access(self):
diff --git a/rpython/rlib/parsing/deterministic.py b/rpython/rlib/parsing/deterministic.py
--- a/rpython/rlib/parsing/deterministic.py
+++ b/rpython/rlib/parsing/deterministic.py
@@ -60,7 +60,8 @@
         self.args = (input, state, source_pos)
 
     def nice_error_message(self, filename="<unknown>"):
-        result = ["  File %s, line %s" % (filename, self.source_pos.lineno)]
+        # + 1 is because source_pos is 0-based and humans 1-based

From noreply at buildbot.pypy.org  Sat Oct 12 01:29:25 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 12 Oct 2013 01:29:25 +0200 (CEST)
Subject: [pypy-commit] pypy remove-intlong-smm: whitespace/unneeded imports
Message-ID: <20131011232925.C9FBF1D22C9@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: remove-intlong-smm
Changeset: r67333:542cae2a1bc7
Date: 2013-10-11 16:27 -0700
http://bitbucket.org/pypy/pypy/changeset/542cae2a1bc7/

Log:	whitespace/unneeded imports

diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -489,7 +489,6 @@
 
 def wrapint(space, x):
     if space.config.objspace.std.withprebuiltint:
-        from pypy.objspace.std.intobject import W_IntObject
         lower = space.config.objspace.std.prebuiltintfrom
         upper =  space.config.objspace.std.prebuiltintto
         # use r_uint to perform a single comparison (this whole function
@@ -507,7 +506,6 @@
         w_res.intval = x
         return w_res
     else:
-        from pypy.objspace.std.intobject import W_IntObject
         return W_IntObject(x)
 
 # ____________________________________________________________
@@ -534,9 +532,8 @@
                              space.wrap(e.msg))
     return space.newlong_from_rbigint(bigint)
 
- at unwrap_spec(w_x = WrappedDefault(0))
+ at unwrap_spec(w_x=WrappedDefault(0))
 def descr__new__(space, w_inttype, w_x, w_base=None):
-    from pypy.objspace.std.intobject import W_IntObject
     w_longval = None
     w_value = w_x     # 'x' is the keyword argument name in CPython
     value = 0

From noreply at buildbot.pypy.org  Sat Oct 12 01:29:27 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 12 Oct 2013 01:29:27 +0200 (CEST)
Subject: [pypy-commit] pypy remove-intlong-smm: most of long's SMM removal
Message-ID: <20131011232927.14C271D22C4@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: remove-intlong-smm
Changeset: r67334:9ea8988dbe08
Date: 2013-10-11 16:28 -0700
http://bitbucket.org/pypy/pypy/changeset/9ea8988dbe08/

Log:	most of long's SMM removal

diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -8,8 +8,8 @@
 from rpython.rlib import jit
 from rpython.rlib.rarithmetic import (
     LONG_BIT, is_valid_int, ovfcheck, string_to_int, r_uint)
+from rpython.rlib.objectmodel import instantiate
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.objectmodel import instantiate
 from rpython.rlib.rstring import ParseStringError, ParseStringOverflowError
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -186,7 +186,7 @@
         return space.newtuple([w(z), w(m)])
 
     @unwrap_spec(w_modulus=WrappedDefault(None))
-    def descr_pow(self, space, w_exponent, w_modulus):
+    def descr_pow(self, space, w_exponent, w_modulus=None):
         if not space.isinstance_w(w_exponent, space.w_int):
             return space.w_NotImplemented
         if space.is_none(w_modulus):
@@ -217,10 +217,11 @@
         return space.pow(w_long1, w_exponent, w_modulus)
 
     @unwrap_spec(w_modulus=WrappedDefault(None))
-    def descr_rpow(self, space, w_base, w_modulus):
+    def descr_rpow(self, space, w_base, w_modulus=None):
         if not space.isinstance_w(w_base, space.w_int):
             return space.w_NotImplemented
-        # XXX: this seems like trouble?
+        # XXX: this seems like trouble?  very likely trouble with int
+        # subclasses implementing __pow__
         return space.pow(w_base, self, w_modulus)
 
     def descr_neg(self, space):
@@ -658,6 +659,7 @@
     __gt__ = interpindirect2app(W_AbstractIntObject.descr_gt),
     __ge__ = interpindirect2app(W_AbstractIntObject.descr_ge),
 
+    # XXX: rtruediv
     __floordiv__ = interpindirect2app(W_AbstractIntObject.descr_floordiv),
     __div__ = interpindirect2app(W_AbstractIntObject.descr_div),
     __truediv__ = interpindirect2app(W_AbstractIntObject.descr_truediv),
diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -3,23 +3,49 @@
 import sys
 
 from rpython.rlib.rbigint import rbigint
+from rpython.rlib.rstring import ParseStringError
+from rpython.tool.sourcetools import func_with_new_name
 
-from pypy.interpreter.error import OperationError
+from pypy.interpreter import typedef
+from pypy.interpreter.error import OperationError, operationerrfmt
+from pypy.interpreter.gateway import (
+    WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
 from pypy.objspace.std import model, newformat
 from pypy.objspace.std.intobject import W_IntObject
-from pypy.objspace.std.longtype import W_AbstractLongObject, long_typedef
-from pypy.objspace.std.model import W_Object, registerimplementation
-from pypy.objspace.std.multimethod import FailedToImplementArgs
+from pypy.objspace.std.model import W_Object
 from pypy.objspace.std.noneobject import W_NoneObject
-from pypy.objspace.std.register_all import register_all
+from pypy.objspace.std.stdtypedef import StdTypeDef
+
+
+class W_AbstractLongObject(W_Object):
+    __slots__ = ()
+
+    def is_w(self, space, w_other):
+        if not isinstance(w_other, W_AbstractLongObject):
+            return False
+        if self.user_overridden_class or w_other.user_overridden_class:
+            return self is w_other
+        return space.bigint_w(self).eq(space.bigint_w(w_other))
+
+    def immutable_unique_id(self, space):
+        if self.user_overridden_class:
+            return None
+        from pypy.objspace.std.model import IDTAG_LONG as tag
+        b = space.bigint_w(self)
+        b = b.lshift(3).or_(rbigint.fromint(tag))
+        return space.newlong_from_rbigint(b)
+
+    def unwrap(w_self, space): #YYYYYY
+        return w_self.longval()
+
+    def int(self, space):
+        raise NotImplementedError
 
 
 class W_LongObject(W_AbstractLongObject):
     """This is a wrapper of rbigint."""
     _immutable_fields_ = ['num']
 
-    typedef = long_typedef
-
     def __init__(self, l):
         self.num = l # instance of rbigint
 
@@ -34,8 +60,8 @@
         try:
             return self.num.tofloat()
         except OverflowError:
-            raise OperationError(space.w_OverflowError,
-                    space.wrap("long int too large to convert to float"))
+            raise operationerrfmt(space.w_OverflowError,
+                                  "long int too large to convert to float")
 
     def toint(self):
         return self.num.toint()
@@ -83,12 +109,278 @@
         try:
             return space.newint(self.num.toint())
         except OverflowError:
-            return long__Long(space, self)
+            return self.descr_long(space)
 
     def __repr__(self):
         return '<W_LongObject(%d)>' % self.num.tolong()
 
-registerimplementation(W_LongObject)
+    def descr_conjugate(self, space):
+        return space.long(self)
+
+    def descr_get_numerator(self, space):
+        return space.long(self)
+
+    def descr_get_denominator(self, space):
+        return space.newlong(1)
+
+    def descr_get_real(self, space):
+        return space.long(self)
+
+    def descr_get_imag(self, space):
+        return space.newlong(0)
+
+    def descr_get_bit_length(self, space):
+        bigint = space.bigint_w(self)
+        try:
+            return space.wrap(bigint.bit_length())
+        except OverflowError:
+            raise OperationError(space.w_OverflowError,
+                                 space.wrap("too many digits in integer"))
+
+    def descr_long(self, space):
+        # long__Long is supposed to do nothing, unless it has a derived
+        # long object, where it should return an exact one.
+        if space.is_w(space.type(self), space.w_long):
+            return self
+        l = self.num
+        return W_LongObject(l)
+    descr_index = func_with_new_name(descr_long, 'descr_index')
+    descr_trunc = func_with_new_name(descr_long, 'descr_trunc')
+    descr_pos = func_with_new_name(descr_long, 'descr_pos')
+
+    def descr_float(self, space):
+        return space.newfloat(self.tofloat(space))
+
+    def descr_repr(self, space):
+        return space.wrap(self.num.repr())
+
+    def descr_str(self, space):
+        return space.wrap(self.num.str())
+
+    def descr_format(self, space, w_format_spec):
+        return newformat.run_formatter(space, w_format_spec,
+                                       "format_int_or_long", self,
+                                       newformat.LONG_KIND)
+
+    def descr_hash(self, space):
+        return space.wrap(self.num.hash())
+
+    def descr_coerce(self, space, w_other):
+        # XXX: consider stian's branch where he optimizes long + ints
+        if space.isinstance_w(w_other, space.w_int):
+            w_other = _delegate_Int2Long(space, w_other)
+        elif not space.isinstance_w(w_other, space.w_long):
+            return space.w_NotImplemented
+        return space.newtuple([self, w_other])
+
+    def _make_descr_cmp(opname):
+        op = getattr(rbigint, opname)
+        def descr_impl(self, space, w_other):
+            if space.isinstance_w(w_other, space.w_int):
+                w_other = _delegate_Int2Long(space, w_other)
+            elif not space.isinstance_w(w_other, space.w_long):
+                return space.w_NotImplemented
+            return space.newbool(op(self.num, w_other.num))
+        return func_with_new_name(descr_impl, "descr_" + opname)
+
+    descr_lt = _make_descr_cmp('lt')
+    descr_le = _make_descr_cmp('le')
+    descr_eq = _make_descr_cmp('eq')
+    descr_ne = _make_descr_cmp('ne')
+    descr_gt = _make_descr_cmp('gt')
+    descr_ge = _make_descr_cmp('ge')
+
+    def _make_descr_binop(opname):
+        from rpython.tool.sourcetools import func_renamer
+        methname = opname + '_' if opname in ('and', 'or') else opname
+        op = getattr(rbigint, methname)
+
+        @func_renamer('descr_' + opname)
+        def descr_binop(self, space, w_other):
+            if space.isinstance_w(w_other, space.w_int):
+                w_other = _delegate_Int2Long(space, w_other)
+            elif not space.isinstance_w(w_other, space.w_long):
+                return space.w_NotImplemented
+            return W_LongObject(op(self.num, w_other.num))
+
+        @func_renamer('descr_r' + opname)
+        def descr_rbinop(self, space, w_other):
+            if space.isinstance_w(w_other, space.w_int):
+                w_other = _delegate_Int2Long(space, w_other)
+            elif not space.isinstance_w(w_other, space.w_long):
+                return space.w_NotImplemented
+            return W_LongObject(op(w_other.num, self.num))
+
+        return descr_binop, descr_rbinop
+
+    descr_add, descr_radd = _make_descr_binop('add')
+    descr_sub, descr_rsub = _make_descr_binop('sub')
+    descr_mul, descr_rmul = _make_descr_binop('mul')
+    descr_and, descr_rand = _make_descr_binop('and')
+    descr_or, descr_ror = _make_descr_binop('or')
+    descr_xor, descr_rxor = _make_descr_binop('xor')
+
+    def _make_descr_unaryop(opname):
+        from rpython.tool.sourcetools import func_renamer
+        op = getattr(rbigint, opname)
+        @func_renamer('descr_' + opname)
+        def descr_unaryop(self, space):
+            return W_LongObject(op(self.num))
+        return descr_unaryop
+
+    descr_neg = _make_descr_unaryop('neg')
+    descr_abs = _make_descr_unaryop('abs')
+    descr_invert = _make_descr_unaryop('invert')
+
+    def descr_oct(self, space):
+        return space.wrap(self.num.oct())
+
+    def descr_hex(self, space):
+        return space.wrap(self.num.hex())
+
+    def descr_nonzero(self, space):
+        return space.newbool(self.num.tobool())
+
+    def descr_lshift(self, space, w_other):
+        if space.isinstance_w(w_other, space.w_int):
+            w_other = _delegate_Int2Long(space, w_other)
+        elif not space.isinstance_w(w_other, space.w_long):
+            return space.w_NotImplemented
+
+        # XXX need to replicate some of the logic, to get the errors right
+        if w_other.num.sign < 0:
+            raise operationerrfmt(space.w_ValueError, "negative shift count")
+        try:
+            shift = w_other.num.toint()
+        except OverflowError:   # b too big
+            raise operationerrfmt(space.w_OverflowError,
+                                  "shift count too large")
+        return W_LongObject(self.num.lshift(shift))
+
+    def descr_rshift(self, space, w_other):
+        if space.isinstance_w(w_other, space.w_int):
+            w_other = _delegate_Int2Long(space, w_other)
+        elif not space.isinstance_w(w_other, space.w_long):
+            return space.w_NotImplemented
+
+        # XXX need to replicate some of the logic, to get the errors right
+        if w_other.num.sign < 0:
+            raise operationerrfmt(space.w_ValueError, "negative shift count")
+        try:
+            shift = w_other.num.toint()
+        except OverflowError:   # b too big # XXX maybe just return 0L instead?
+            raise operationerrfmt(space.w_OverflowError,
+                                  "shift count too large")
+        return newlong(space, self.num.rshift(shift))
+
+    # XXX: need rtruediv etc
+    def descr_truediv(self, space, w_other):
+        if space.isinstance_w(w_other, space.w_int):
+            w_other = _delegate_Int2Long(space, w_other)
+        elif not space.isinstance_w(w_other, space.w_long):
+            return space.w_NotImplemented
+
+        try:
+            f = self.num.truediv(w_other.num)
+        except ZeroDivisionError:
+            raise operationerrfmt(space.w_ZeroDivisionError,
+                                  "long division or modulo by zero")
+        except OverflowError:
+            raise operationerrfmt(space.w_OverflowError,
+                                  "long/long too large for a float")
+        return space.newfloat(f)
+
+    def descr_floordiv(self, space, w_other):
+        if space.isinstance_w(w_other, space.w_int):
+            w_other = _delegate_Int2Long(space, w_other)
+        elif not space.isinstance_w(w_other, space.w_long):
+            return space.w_NotImplemented
+
+        try:
+            z = self.num.floordiv(w_other.num)
+        except ZeroDivisionError:
+            raise operationerrfmt(space.w_ZeroDivisionError,
+                                  "long division or modulo by zero")
+        return newlong(space, z)
+
+    def descr_div(self, space, w_other):
+        if space.isinstance_w(w_other, space.w_int):
+            w_other = _delegate_Int2Long(space, w_other)
+        elif not space.isinstance_w(w_other, space.w_long):
+            return space.w_NotImplemented
+
+        return self.floordiv(space, w_other)
+
+    def descr_mod(self, space, w_other):
+        if space.isinstance_w(w_other, space.w_int):
+            w_other = _delegate_Int2Long(space, w_other)
+        elif not space.isinstance_w(w_other, space.w_long):
+            return space.w_NotImplemented
+
+        try:
+            z = self.num.mod(w_other.num)
+        except ZeroDivisionError:
+            raise operationerrfmt(space.w_ZeroDivisionError,
+                                  "long division or modulo by zero")
+        return newlong(space, z)
+
+    def descr_divmod(self, space, w_other):
+        if space.isinstance_w(w_other, space.w_int):
+            w_other = _delegate_Int2Long(space, w_other)
+        elif not space.isinstance_w(w_other, space.w_long):
+            return space.w_NotImplemented
+
+        try:
+            div, mod = self.num.divmod(w_other.num)
+        except ZeroDivisionError:
+            raise operationerrfmt(space.w_ZeroDivisionError,
+                                  "long division or modulo by zero")
+        return space.newtuple([newlong(space, div), newlong(space, mod)])
+
+    @unwrap_spec(w_modulus=WrappedDefault(None))
+    def descr_pow(self, space, w_exponent, w_modulus=None):
+        if space.isinstance_w(w_exponent, space.w_int):
+            w_exponent = _delegate_Int2Long(space, w_exponent)
+        elif not space.isinstance_w(w_exponent, space.w_long):
+            return space.w_NotImplemented
+        if space.isinstance_w(w_modulus, space.w_int):
+            w_modulus = _delegate_Int2Long(space, w_modulus)
+        elif space.is_none(w_modulus):
+            # XXX need to replicate some of the logic, to get the errors right
+            if w_exponent.num.sign < 0:
+                return space.pow(self.descr_float(space), w_exponent, w_modulus)
+            return W_LongObject(self.num.pow(w_exponent.num, None))
+        elif not space.isinstance_w(w_modulus, space.w_long):
+            return space.w_NotImplemented
+
+        # XXX need to replicate some of the logic, to get the errors right
+        if w_exponent.num.sign < 0:
+            raise OperationError(
+                space.w_TypeError,
+                space.wrap(
+                    "pow() 2nd argument "
+                    "cannot be negative when 3rd argument specified"))
+        try:
+            return W_LongObject(self.num.pow(w_exponent.num, w_modulus.num))
+        except ValueError:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("pow 3rd argument cannot be 0"))
+
+    @unwrap_spec(w_modulus=WrappedDefault(None))
+    def descr_rpow(self, space, w_exponent, w_modulus=None):
+        if space.isinstance_w(w_exponent, space.w_int):
+            w_exponent = _delegate_Int2Long(space, w_exponent)
+        elif not space.isinstance_w(w_exponent, space.w_long):
+            return space.w_NotImplemented
+        ### XXX: these may needs all the checks above has. annoying
+        #if not space.isinstance_w(w_exponent, space.w_long):
+        #    return space.w_NotImplemented
+        # XXX:
+        return space.pow(w_exponent, self, w_modulus)
+
+    def descr_getnewargs(self, space):
+        return space.newtuple([W_LongObject(self.num)])
+
 
 def newlong(space, bigint):
     """Turn the bigint into a W_LongObject.  If withsmalllong is enabled,
@@ -106,220 +398,10 @@
     return W_LongObject(bigint)
 
 
-# bool-to-long
-def delegate_Bool2Long(space, w_bool):
-    return W_LongObject(rbigint.frombool(space.is_true(w_bool)))
+def _delegate_Int2Long(space, w_intobj):
+    """int-to-long delegation"""
+    return W_LongObject.fromint(space, w_intobj.int_w(space))
 
-# int-to-long delegation
-def delegate_Int2Long(space, w_intobj):
-    return W_LongObject.fromint(space, w_intobj.intval)
-
-
-# long__Long is supposed to do nothing, unless it has
-# a derived long object, where it should return
-# an exact one.
-def long__Long(space, w_long1):
-    if space.is_w(space.type(w_long1), space.w_long):
-        return w_long1
-    l = w_long1.num
-    return W_LongObject(l)
-trunc__Long = long__Long
-
-def long__Int(space, w_intobj):
-    return space.newlong(w_intobj.intval)
-
-def index__Long(space, w_value):
-    return long__Long(space, w_value)
-
-def float__Long(space, w_longobj):
-    return space.newfloat(w_longobj.tofloat(space))
-
-def repr__Long(space, w_long):
-    return space.wrap(w_long.num.repr())
-
-def str__Long(space, w_long):
-    return space.wrap(w_long.num.str())
-
-def format__Long_ANY(space, w_long, w_format_spec):
-    return newformat.run_formatter(space, w_format_spec, "format_int_or_long",
-                                   w_long, newformat.LONG_KIND)
-
-
-def lt__Long_Long(space, w_long1, w_long2):
-    return space.newbool(w_long1.num.lt(w_long2.num))
-def le__Long_Long(space, w_long1, w_long2):
-    return space.newbool(w_long1.num.le(w_long2.num))
-def eq__Long_Long(space, w_long1, w_long2):
-    return space.newbool(w_long1.num.eq(w_long2.num))
-def ne__Long_Long(space, w_long1, w_long2):
-    return space.newbool(w_long1.num.ne(w_long2.num))
-def gt__Long_Long(space, w_long1, w_long2):
-    return space.newbool(w_long1.num.gt(w_long2.num))
-def ge__Long_Long(space, w_long1, w_long2):
-    return space.newbool(w_long1.num.ge(w_long2.num))
-
-def lt__Long_Int(space, w_long1, w_int2):
-    return space.newbool(w_long1.num.lt(rbigint.fromint(w_int2.intval)))
-def le__Long_Int(space, w_long1, w_int2):
-    return space.newbool(w_long1.num.le(rbigint.fromint(w_int2.intval)))
-def eq__Long_Int(space, w_long1, w_int2):
-    return space.newbool(w_long1.num.eq(rbigint.fromint(w_int2.intval)))
-def ne__Long_Int(space, w_long1, w_int2):
-    return space.newbool(w_long1.num.ne(rbigint.fromint(w_int2.intval)))
-def gt__Long_Int(space, w_long1, w_int2):
-    return space.newbool(w_long1.num.gt(rbigint.fromint(w_int2.intval)))
-def ge__Long_Int(space, w_long1, w_int2):
-    return space.newbool(w_long1.num.ge(rbigint.fromint(w_int2.intval)))
-
-def lt__Int_Long(space, w_int1, w_long2):
-    return space.newbool(rbigint.fromint(w_int1.intval).lt(w_long2.num))
-def le__Int_Long(space, w_int1, w_long2):
-    return space.newbool(rbigint.fromint(w_int1.intval).le(w_long2.num))
-def eq__Int_Long(space, w_int1, w_long2):
-    return space.newbool(rbigint.fromint(w_int1.intval).eq(w_long2.num))
-def ne__Int_Long(space, w_int1, w_long2):
-    return space.newbool(rbigint.fromint(w_int1.intval).ne(w_long2.num))
-def gt__Int_Long(space, w_int1, w_long2):
-    return space.newbool(rbigint.fromint(w_int1.intval).gt(w_long2.num))
-def ge__Int_Long(space, w_int1, w_long2):
-    return space.newbool(rbigint.fromint(w_int1.intval).ge(w_long2.num))
-
-
-def hash__Long(space, w_value):
-    return space.wrap(w_value.num.hash())
-
-# coerce
-def coerce__Long_Long(space, w_long1, w_long2):
-    return space.newtuple([w_long1, w_long2])
-
-
-def add__Long_Long(space, w_long1, w_long2):
-    return W_LongObject(w_long1.num.add(w_long2.num))
-
-def sub__Long_Long(space, w_long1, w_long2):
-    return W_LongObject(w_long1.num.sub(w_long2.num))
-
-def mul__Long_Long(space, w_long1, w_long2):
-    return W_LongObject(w_long1.num.mul(w_long2.num))
-
-def truediv__Long_Long(space, w_long1, w_long2):
-    try:
-        f = w_long1.num.truediv(w_long2.num)
-    except ZeroDivisionError:
-        raise OperationError(space.w_ZeroDivisionError,
-                             space.wrap("long division or modulo by zero"))
-    except OverflowError:
-        raise OperationError(space.w_OverflowError,
-                             space.wrap("long/long too large for a float"))
-    return space.newfloat(f)
-
-def floordiv__Long_Long(space, w_long1, w_long2):
-    try:
-        z = w_long1.num.floordiv(w_long2.num)
-    except ZeroDivisionError:
-        raise OperationError(space.w_ZeroDivisionError,
-                             space.wrap("long division or modulo by zero"))
-    return newlong(space, z)
-
-def div__Long_Long(space, w_long1, w_long2):
-    return floordiv__Long_Long(space, w_long1, w_long2)
-
-def mod__Long_Long(space, w_long1, w_long2):
-    try:
-        z = w_long1.num.mod(w_long2.num)
-    except ZeroDivisionError:
-        raise OperationError(space.w_ZeroDivisionError,
-                             space.wrap("long division or modulo by zero"))
-    return newlong(space, z)
-
-def divmod__Long_Long(space, w_long1, w_long2):
-    try:
-        div, mod = w_long1.num.divmod(w_long2.num)
-    except ZeroDivisionError:
-        raise OperationError(space.w_ZeroDivisionError,
-                             space.wrap("long division or modulo by zero"))
-    return space.newtuple([newlong(space, div), newlong(space, mod)])
-
-def pow__Long_Long_Long(space, w_long1, w_long2, w_long3):
-    # XXX need to replicate some of the logic, to get the errors right
-    if w_long2.num.sign < 0:
-        raise OperationError(
-            space.w_TypeError,
-            space.wrap(
-                "pow() 2nd argument "
-                "cannot be negative when 3rd argument specified"))
-    try:
-        return W_LongObject(w_long1.num.pow(w_long2.num, w_long3.num))
-    except ValueError:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("pow 3rd argument cannot be 0"))
-
-def pow__Long_Long_None(space, w_long1, w_long2, w_long3):
-    # XXX need to replicate some of the logic, to get the errors right
-    if w_long2.num.sign < 0:
-        raise FailedToImplementArgs(
-            space.w_ValueError,
-            space.wrap("long pow() too negative"))
-    return W_LongObject(w_long1.num.pow(w_long2.num, None))
-
-def neg__Long(space, w_long1):
-    return W_LongObject(w_long1.num.neg())
-
-def pos__Long(space, w_long):
-    return long__Long(space, w_long)
-
-def abs__Long(space, w_long):
-    return W_LongObject(w_long.num.abs())
-
-def nonzero__Long(space, w_long):
-    return space.newbool(w_long.num.tobool())
-
-def invert__Long(space, w_long):
-    return W_LongObject(w_long.num.invert())
-
-def lshift__Long_Long(space, w_long1, w_long2):
-    # XXX need to replicate some of the logic, to get the errors right
-    if w_long2.num.sign < 0:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("negative shift count"))
-    try:
-        shift = w_long2.num.toint()
-    except OverflowError:   # b too big
-        raise OperationError(space.w_OverflowError,
-                             space.wrap("shift count too large"))
-    return W_LongObject(w_long1.num.lshift(shift))
-
-def rshift__Long_Long(space, w_long1, w_long2):
-    # XXX need to replicate some of the logic, to get the errors right
-    if w_long2.num.sign < 0:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("negative shift count"))
-    try:
-        shift = w_long2.num.toint()
-    except OverflowError:   # b too big # XXX maybe just return 0L instead?
-        raise OperationError(space.w_OverflowError,
-                             space.wrap("shift count too large"))
-    return newlong(space, w_long1.num.rshift(shift))
-
-def and__Long_Long(space, w_long1, w_long2):
-    return newlong(space, w_long1.num.and_(w_long2.num))
-
-def xor__Long_Long(space, w_long1, w_long2):
-    return W_LongObject(w_long1.num.xor(w_long2.num))
-
-def or__Long_Long(space, w_long1, w_long2):
-    return W_LongObject(w_long1.num.or_(w_long2.num))
-
-def oct__Long(space, w_long1):
-    return space.wrap(w_long1.num.oct())
-
-def hex__Long(space, w_long1):
-    return space.wrap(w_long1.num.hex())
-
-def getnewargs__Long(space, w_long1):
-    return space.newtuple([W_LongObject(w_long1.num)])
-
-register_all(vars())
 
 # register implementations of ops that recover int op overflows
 def recover_with_smalllong(space):
@@ -327,6 +409,7 @@
     return (space.config.objspace.std.withsmalllong and
             sys.maxint == 2147483647)
 
+# XXX:
 # binary ops
 for opname in ['add', 'sub', 'mul', 'div', 'floordiv', 'truediv', 'mod',
                'divmod', 'lshift']:
@@ -335,8 +418,8 @@
     if recover_with_smalllong(space) and %(opname)r != 'truediv':
         from pypy.objspace.std.smalllongobject import %(opname)s_ovr
         return %(opname)s_ovr(space, w_int1, w_int2)
-    w_long1 = delegate_Int2Long(space, w_int1)
-    w_long2 = delegate_Int2Long(space, w_int2)
+    w_long1 = _delegate_Int2Long(space, w_int1)
+    w_long2 = _delegate_Int2Long(space, w_int2)
     return %(opname)s__Long_Long(space, w_long1, w_long2)
 """ % {'opname': opname}, '', 'exec')
 
@@ -350,7 +433,7 @@
     if recover_with_smalllong(space):
         from pypy.objspace.std.smalllongobject import %(opname)s_ovr
         return %(opname)s_ovr(space, w_int1)
-    w_long1 = delegate_Int2Long(space, w_int1)
+    w_long1 = _delegate_Int2Long(space, w_int1)
     return %(opname)s__Long(space, w_long1)
 """ % {'opname': opname}
 
@@ -362,13 +445,13 @@
     if recover_with_smalllong(space):
         from pypy.objspace.std.smalllongobject import pow_ovr
         return pow_ovr(space, w_int1, w_int2)
-    w_long1 = delegate_Int2Long(space, w_int1)
-    w_long2 = delegate_Int2Long(space, w_int2)
+    w_long1 = _delegate_Int2Long(space, w_int1)
+    w_long2 = _delegate_Int2Long(space, w_int2)
     return pow__Long_Long_None(space, w_long1, w_long2, w_none3)
 
 def pow_ovr__Int_Int_Long(space, w_int1, w_int2, w_long3):
-    w_long1 = delegate_Int2Long(space, w_int1)
-    w_long2 = delegate_Int2Long(space, w_int2)
+    w_long1 = _delegate_Int2Long(space, w_int1)
+    w_long2 = _delegate_Int2Long(space, w_int2)
     return pow__Long_Long_Long(space, w_long1, w_long2, w_long3)
 
 model.MM.pow.register(pow_ovr__Int_Int_None, W_IntObject, W_IntObject,
@@ -377,3 +460,155 @@
                       W_LongObject, order=1)
 
 
+ at unwrap_spec(w_x=WrappedDefault(0))
+def descr__new__(space, w_longtype, w_x, w_base=None):
+    if space.config.objspace.std.withsmalllong:
+        from pypy.objspace.std.smalllongobject import W_SmallLongObject
+    else:
+        W_SmallLongObject = None
+
+    w_value = w_x     # 'x' is the keyword argument name in CPython
+    if w_base is None:
+        # check for easy cases
+        if (W_SmallLongObject and type(w_value) is W_SmallLongObject
+            and space.is_w(w_longtype, space.w_long)):
+            return w_value
+        elif type(w_value) is W_LongObject:
+            return newbigint(space, w_longtype, w_value.num)
+        elif space.isinstance_w(w_value, space.w_str):
+            return string_to_w_long(space, w_longtype, space.str_w(w_value))
+        elif space.isinstance_w(w_value, space.w_unicode):
+            from pypy.objspace.std.unicodeobject import unicode_to_decimal_w
+            return string_to_w_long(space, w_longtype,
+                                    unicode_to_decimal_w(space, w_value))
+        else:
+            # otherwise, use the __long__() or the __trunc__ methods
+            w_obj = w_value
+            if (space.lookup(w_obj, '__long__') is not None or
+                space.lookup(w_obj, '__int__') is not None):
+                w_obj = space.long(w_obj)
+            else:
+                w_obj = space.trunc(w_obj)
+                # :-(  blame CPython 2.7
+                if space.lookup(w_obj, '__long__') is not None:
+                    w_obj = space.long(w_obj)
+                else:
+                    w_obj = space.int(w_obj)
+            bigint = space.bigint_w(w_obj)
+            return newbigint(space, w_longtype, bigint)
+    else:
+        base = space.int_w(w_base)
+
+        if space.isinstance_w(w_value, space.w_unicode):
+            from pypy.objspace.std.unicodeobject import unicode_to_decimal_w
+            s = unicode_to_decimal_w(space, w_value)
+        else:
+            try:
+                s = space.str_w(w_value)
+            except OperationError:
+                msg = "long() can't convert non-string with explicit base"
+                raise operationerrfmt(space.w_TypeError, msg)
+        return string_to_w_long(space, w_longtype, s, base)
+
+
+def string_to_w_long(space, w_longtype, s, base=10):
+    try:
+        bigint = rbigint.fromstr(s, base)
+    except ParseStringError as e:
+        raise operationerrfmt(space.w_ValueError, e.msg)
+    return newbigint(space, w_longtype, bigint)
+string_to_w_long._dont_inline_ = True
+
+
+def newbigint(space, w_longtype, bigint):
+    """Turn the bigint into a W_LongObject.  If withsmalllong is enabled,
+    check if the bigint would fit in a smalllong, and return a
+    W_SmallLongObject instead if it does.  Similar to newlong() in
+    longobject.py, but takes an explicit w_longtype argument.
+    """
+    if (space.config.objspace.std.withsmalllong
+        and space.is_w(w_longtype, space.w_long)):
+        try:
+            z = bigint.tolonglong()
+        except OverflowError:
+            pass
+        else:
+            from pypy.objspace.std.smalllongobject import W_SmallLongObject
+            return W_SmallLongObject(z)
+    w_obj = space.allocate_instance(W_LongObject, w_longtype)
+    W_LongObject.__init__(w_obj, bigint)
+    return w_obj
+
+
+W_LongObject.typedef = StdTypeDef("long",
+    __doc__ = """long(x[, base]) -> integer
+
+Convert a string or number to a long integer, if possible.  A floating
+point argument will be truncated towards zero (this does not include a
+string representation of a floating point number!)  When converting a
+string, use the optional base.  It is an error to supply a base when
+converting a non-string.""",
+    __new__ = interp2app(descr__new__),
+    conjugate = interp2app(W_LongObject.descr_conjugate),
+    numerator = typedef.GetSetProperty(W_LongObject.descr_get_numerator),
+    denominator = typedef.GetSetProperty(W_LongObject.descr_get_denominator),
+    real = typedef.GetSetProperty(W_LongObject.descr_get_real),
+    imag = typedef.GetSetProperty(W_LongObject.descr_get_imag),
+    bit_length = interp2app(W_LongObject.descr_get_bit_length),
+
+    # XXX: likely need indirect everything for SmallLong
+    __int__ = interpindirect2app(W_AbstractLongObject.int),
+    __long__ = interp2app(W_LongObject.descr_long),
+    __trunc__ = interp2app(W_LongObject.descr_trunc),
+    __index__ = interp2app(W_LongObject.descr_index),
+    __float__ = interp2app(W_LongObject.descr_float),
+    __repr__ = interp2app(W_LongObject.descr_repr),
+    __str__ = interp2app(W_LongObject.descr_str),
+    __format__ = interp2app(W_LongObject.descr_format),
+
+    __hash__ = interp2app(W_LongObject.descr_hash),
+    __coerce__ = interp2app(W_LongObject.descr_coerce),
+
+    __lt__ = interp2app(W_LongObject.descr_lt),
+    __le__ = interp2app(W_LongObject.descr_le),
+    __eq__ = interp2app(W_LongObject.descr_eq),
+    __ne__ = interp2app(W_LongObject.descr_ne),
+    __gt__ = interp2app(W_LongObject.descr_gt),
+    __ge__ = interp2app(W_LongObject.descr_ge),
+
+    __add__ = interp2app(W_LongObject.descr_add),
+    __radd__ = interp2app(W_LongObject.descr_radd),
+    __sub__ = interp2app(W_LongObject.descr_sub),
+    __rsub__ = interp2app(W_LongObject.descr_rsub),
+    __mul__ = interp2app(W_LongObject.descr_mul),
+    __rmul__ = interp2app(W_LongObject.descr_rmul),
+
+    __and__ = interp2app(W_LongObject.descr_and),
+    __rand__ = interp2app(W_LongObject.descr_rand),
+    __or__ = interp2app(W_LongObject.descr_or),
+    __ror__ = interp2app(W_LongObject.descr_ror),
+    __xor__ = interp2app(W_LongObject.descr_xor),
+    __rxor__ = interp2app(W_LongObject.descr_rxor),
+
+    __neg__ = interp2app(W_LongObject.descr_neg),
+    __pos__ = interp2app(W_LongObject.descr_pos),
+    __abs__ = interp2app(W_LongObject.descr_abs),
+    __nonzero__ = interp2app(W_LongObject.descr_nonzero),
+    __invert__ = interp2app(W_LongObject.descr_invert),
+    __oct__ = interp2app(W_LongObject.descr_oct),
+    __hex__ = interp2app(W_LongObject.descr_hex),
+
+    __lshift__ = interp2app(W_LongObject.descr_lshift),
+    __rshift__ = interp2app(W_LongObject.descr_rshift),
+
+    __truediv__ = interp2app(W_LongObject.descr_truediv),
+    __floordiv__ = interp2app(W_LongObject.descr_floordiv),
+    __div__ = interp2app(W_LongObject.descr_div),
+    __mod__ = interp2app(W_LongObject.descr_mod),
+    __divmod__ = interp2app(W_LongObject.descr_divmod),
+
+    __pow__ = interp2app(W_LongObject.descr_pow),
+    __rpow__ = interp2app(W_LongObject.descr_rpow),
+
+    __getnewargs__ = interp2app(W_LongObject.descr_getnewargs),
+)
diff --git a/pypy/objspace/std/longtype.py b/pypy/objspace/std/longtype.py
deleted file mode 100644
--- a/pypy/objspace/std/longtype.py
+++ /dev/null
@@ -1,159 +0,0 @@
-from pypy.interpreter.error import OperationError
-from pypy.interpreter import typedef
-from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault,\
-     interpindirect2app
-from pypy.objspace.std.model import W_Object
-from pypy.objspace.std.stdtypedef import StdTypeDef
-from rpython.rlib.rstring import ParseStringError
-from rpython.rlib.rbigint import rbigint
-
-def descr_conjugate(space, w_int):
-    return space.long(w_int)
-
-
- at unwrap_spec(w_x = WrappedDefault(0))
-def descr__new__(space, w_longtype, w_x, w_base=None):
-    from pypy.objspace.std.longobject import W_LongObject
-    if space.config.objspace.std.withsmalllong:
-        from pypy.objspace.std.smalllongobject import W_SmallLongObject
-    else:
-        W_SmallLongObject = None
-
-    w_value = w_x     # 'x' is the keyword argument name in CPython
-    if w_base is None:
-        # check for easy cases
-        if (W_SmallLongObject and type(w_value) is W_SmallLongObject
-            and space.is_w(w_longtype, space.w_long)):
-            return w_value
-        elif type(w_value) is W_LongObject:
-            return newbigint(space, w_longtype, w_value.num)
-        elif space.isinstance_w(w_value, space.w_str):
-            return string_to_w_long(space, w_longtype, space.str_w(w_value))
-        elif space.isinstance_w(w_value, space.w_unicode):
-            from pypy.objspace.std.unicodeobject import unicode_to_decimal_w
-            return string_to_w_long(space, w_longtype,
-                                    unicode_to_decimal_w(space, w_value))
-        else:
-            # otherwise, use the __long__() or the __trunc__ methods
-            w_obj = w_value
-            if (space.lookup(w_obj, '__long__') is not None or
-                space.lookup(w_obj, '__int__') is not None):
-                w_obj = space.long(w_obj)
-            else:
-                w_obj = space.trunc(w_obj)
-                # :-(  blame CPython 2.7
-                if space.lookup(w_obj, '__long__') is not None:
-                    w_obj = space.long(w_obj)
-                else:
-                    w_obj = space.int(w_obj)
-            bigint = space.bigint_w(w_obj)
-            return newbigint(space, w_longtype, bigint)
-    else:
-        base = space.int_w(w_base)
-
-        if space.isinstance_w(w_value, space.w_unicode):
-            from pypy.objspace.std.unicodeobject import unicode_to_decimal_w
-            s = unicode_to_decimal_w(space, w_value)
-        else:
-            try:
-                s = space.str_w(w_value)
-            except OperationError:
-                raise OperationError(space.w_TypeError,
-                                     space.wrap("long() can't convert non-string "
-                                                "with explicit base"))
-        return string_to_w_long(space, w_longtype, s, base)
-
-
-def string_to_w_long(space, w_longtype, s, base=10):
-    try:
-        bigint = rbigint.fromstr(s, base)
-    except ParseStringError, e:
-        raise OperationError(space.w_ValueError,
-                             space.wrap(e.msg))
-    return newbigint(space, w_longtype, bigint)
-string_to_w_long._dont_inline_ = True
-
-def newbigint(space, w_longtype, bigint):
-    """Turn the bigint into a W_LongObject.  If withsmalllong is enabled,
-    check if the bigint would fit in a smalllong, and return a
-    W_SmallLongObject instead if it does.  Similar to newlong() in
-    longobject.py, but takes an explicit w_longtype argument.
-    """
-    if (space.config.objspace.std.withsmalllong
-        and space.is_w(w_longtype, space.w_long)):
-        try:
-            z = bigint.tolonglong()
-        except OverflowError:
-            pass
-        else:
-            from pypy.objspace.std.smalllongobject import W_SmallLongObject
-            return W_SmallLongObject(z)
-    from pypy.objspace.std.longobject import W_LongObject
-    w_obj = space.allocate_instance(W_LongObject, w_longtype)
-    W_LongObject.__init__(w_obj, bigint)
-    return w_obj
-
-def descr_get_numerator(space, w_obj):
-    return space.long(w_obj)
-
-def descr_get_denominator(space, w_obj):
-    return space.newlong(1)
-
-def descr_get_real(space, w_obj):
-    return space.long(w_obj)
-
-def descr_get_imag(space, w_obj):
-    return space.newlong(0)
-
-def bit_length(space, w_obj):
-    bigint = space.bigint_w(w_obj)
-    try:
-        return space.wrap(bigint.bit_length())
-    except OverflowError:
-        raise OperationError(space.w_OverflowError,
-                             space.wrap("too many digits in integer"))
-
-# ____________________________________________________________
-
-class W_AbstractLongObject(W_Object):
-    __slots__ = ()
-
-    def is_w(self, space, w_other):
-        if not isinstance(w_other, W_AbstractLongObject):
-            return False
-        if self.user_overridden_class or w_other.user_overridden_class:
-            return self is w_other
-        return space.bigint_w(self).eq(space.bigint_w(w_other))
-
-    def immutable_unique_id(self, space):
-        if self.user_overridden_class:
-            return None
-        from pypy.objspace.std.model import IDTAG_LONG as tag
-        b = space.bigint_w(self)
-        b = b.lshift(3).or_(rbigint.fromint(tag))
-        return space.newlong_from_rbigint(b)
-
-    def unwrap(w_self, space): #YYYYYY
-        return w_self.longval()
-
-    def int(self, space):
-        raise NotImplementedError
-
-long_typedef = StdTypeDef("long",
-    __doc__ = '''long(x[, base]) -> integer
-
-Convert a string or number to a long integer, if possible.  A floating
-point argument will be truncated towards zero (this does not include a
-string representation of a floating point number!)  When converting a
-string, use the optional base.  It is an error to supply a base when
-converting a non-string.''',
-    __new__ = interp2app(descr__new__),
-    conjugate = interp2app(descr_conjugate),
-    numerator = typedef.GetSetProperty(descr_get_numerator),
-    denominator = typedef.GetSetProperty(descr_get_denominator),
-    real = typedef.GetSetProperty(descr_get_real),
-    imag = typedef.GetSetProperty(descr_get_imag),
-    bit_length = interp2app(bit_length),
-    __int__ = interpindirect2app(W_AbstractLongObject.int),
-)
-long_typedef.registermethods(globals())
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -39,7 +39,7 @@
             from pypy.objspace.std.bytearraytype import bytearray_typedef
             from pypy.objspace.std.typeobject   import type_typedef
             from pypy.objspace.std.slicetype  import slice_typedef
-            from pypy.objspace.std.longtype   import long_typedef
+            #from pypy.objspace.std.longtype   import long_typedef
             from pypy.objspace.std.unicodetype import unicode_typedef
             from pypy.objspace.std.nonetype import none_typedef
         self.pythontypes = [value for key, value in result.__dict__.items()
@@ -81,6 +81,7 @@
         self.pythontypes.append(iterobject.W_AbstractSeqIterObject.typedef)
         self.pythontypes.append(intobject.W_IntObject.typedef)
         self.pythontypes.append(boolobject.W_BoolObject.typedef)
+        self.pythontypes.append(longobject.W_LongObject.typedef)
 
         # the set of implementation types
         self.typeorder = {
@@ -107,6 +108,7 @@
             if option.startswith("with") and option in option_to_typename:
                 for classname in option_to_typename[option]:
                     modname = classname[:classname.index('.')]
+                    if modname == 'smalllongobject': continue # XXX:
                     classname = classname[classname.index('.')+1:]
                     d = {}
                     exec "from pypy.objspace.std.%s import %s" % (
@@ -136,7 +138,7 @@
         self.typeorder[boolobject.W_BoolObject] += [
 #            (intobject.W_IntObject,     boolobject.delegate_Bool2IntObject),
             (floatobject.W_FloatObject, floatobject.delegate_Bool2Float),
-            (longobject.W_LongObject,   longobject.delegate_Bool2Long),
+#            (longobject.W_LongObject,   longobject.delegate_Bool2Long),
             (complexobject.W_ComplexObject, complexobject.delegate_Bool2Complex),
             ]
         self.typeorder[intobject.W_IntObject] += [
@@ -144,7 +146,7 @@
             (longobject.W_LongObject,   longobject.delegate_Int2Long),
             (complexobject.W_ComplexObject, complexobject.delegate_Int2Complex),
             ]
-        if config.objspace.std.withsmalllong:
+        if False and config.objspace.std.withsmalllong:
             from pypy.objspace.std import smalllongobject
             self.typeorder[boolobject.W_BoolObject] += [
                 (smalllongobject.W_SmallLongObject, smalllongobject.delegate_Bool2SmallLong),

From noreply at buildbot.pypy.org  Sat Oct 12 18:23:50 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Sat, 12 Oct 2013 18:23:50 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: attempt to fix for trying to patch the
 JMP target of a guard twice
Message-ID: <20131012162350.8BCC91C0203@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67335:6e93c32df66d
Date: 2013-10-12 18:22 +0200
http://bitbucket.org/pypy/pypy/changeset/6e93c32df66d/

Log:	attempt to fix for trying to patch the JMP target of a guard twice

diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -205,6 +205,13 @@
         deadframe = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe)
         return deadframe.jf_savedata
 
+    def guard_already_patched(self, faildescr):
+        # returns True if the guard jump target is already patched
+        # to point to a bridge
+        raise NotImplemented
+
+
+
     def free_loop_and_bridges(self, compiled_loop_token):
         AbstractCPU.free_loop_and_bridges(self, compiled_loop_token)
         blocks = compiled_loop_token.asmmemmgr_blocks
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -104,6 +104,11 @@
                                               original_loop_token, log=log,
                                               logger=logger)
 
+    def guard_already_patched(self, faildescr):
+        # only needed for STM so far
+        return faildescr._x86_adr_jump_offset == 0
+
+        
     def clear_latest_values(self, count):
         setitem = self.assembler.fail_boxes_ptr.setitem
         null = lltype.nullptr(llmemory.GCREF.TO)
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -573,7 +573,16 @@
         with stm_ignored:
             approx_counter = self._counter + 1
             self._counter = approx_counter
-        return approx_counter >= trace_eagerness and not self.rd_stm_busy
+
+        # The call to guard_already_patched is necessary because it is 
+        # possible that the current transaction didn't see the 
+        # patched JMP yet, but already sees rd_stm_busy as False (because
+        # the patching is in raw-memory).
+        # Thus it may try to compile a trace too and also patch the assembler.
+        # However, this would trigger the assertion in 
+        #     x86.assembler.patch_jump_for_descr.
+        return (approx_counter >= trace_eagerness and not self.rd_stm_busy
+                and not metainterp_sd.cpu.guard_already_patched(self))
 
     def must_compile_nonstm(self, deadframe, metainterp_sd, jitdriver_sd):
         trace_eagerness = jitdriver_sd.warmstate.trace_eagerness

From noreply at buildbot.pypy.org  Sat Oct 12 19:30:18 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 12 Oct 2013 19:30:18 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Tweak tweak
Message-ID: <20131012173018.2177D1C01F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67336:b4f9c1401b1f
Date: 2013-10-12 19:29 +0200
http://bitbucket.org/pypy/pypy/changeset/b4f9c1401b1f/

Log:	Tweak tweak

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1761,7 +1761,7 @@
             debug_print("number of objects to mark",
                         self.objects_to_trace.length())
 
-            estimate = self.nursery_size    # XXX
+            estimate = self.nursery_size // 10    # XXX
             self.visit_all_objects_step(estimate)
 
             # XXX A simplifying assumption that should be checked,

From noreply at buildbot.pypy.org  Sat Oct 12 19:31:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 12 Oct 2013 19:31:02 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Change the default GC for
	benchmarks
Message-ID: <20131012173102.54D771C01F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67337:04b6648b330c
Date: 2013-10-12 19:30 +0200
http://bitbucket.org/pypy/pypy/changeset/04b6648b330c/

Log:	Change the default GC for benchmarks

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -11,7 +11,7 @@
 DEFL_CLEVER_MALLOC_REMOVAL_INLINE_THRESHOLD = 32.4
 DEFL_LOW_INLINE_THRESHOLD = DEFL_INLINE_THRESHOLD / 2.0
 
-DEFL_GC = "minimark"
+DEFL_GC = "incminimark"   # XXX
 
 if sys.platform.startswith("linux"):
     DEFL_ROOTFINDER_WITHJIT = "asmgcc"

From noreply at buildbot.pypy.org  Sun Oct 13 00:55:49 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 13 Oct 2013 00:55:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: support more numpy c-api
Message-ID: <20131012225549.C29071C0203@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67338:a30928d7419e
Date: 2013-10-13 01:52 +0300
http://bitbucket.org/pypy/pypy/changeset/a30928d7419e/

Log:	support more numpy c-api

diff --git a/pypy/module/cpyext/include/numpy/npy_3kcompat.h b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
--- a/pypy/module/cpyext/include/numpy/npy_3kcompat.h
+++ b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
@@ -0,0 +1,39 @@
+/*
+ * In numpy this is a convenience header file providing compatibility utilities
+ * for supporting Python 2 and Python 3 in the same code base.
+ *
+ * PyPy uses it as a convenient place to add compatability declarations
+ */
+
+#ifndef _NPY_3KCOMPAT_H_
+#define _NPY_3KCOMPAT_H_
+
+#include <numpy/npy_common.h>
+
+#define npy_PyFile_Dup(file, mode) (NULL)
+#define npy_PyFile_DupClose(file, handle) (0)
+
+static NPY_INLINE PyObject*
+npy_PyFile_OpenFile(PyObject *filename, const char *mode)
+{
+    PyObject *open;
+    open = PyDict_GetItemString(PyEval_GetBuiltins(), "open");
+    if (open == NULL) {
+        return NULL;
+    }
+    return PyObject_CallFunction(open, "Os", filename, mode);
+}
+
+static NPY_INLINE int
+npy_PyFile_CloseFile(PyObject *file)
+{
+    PyObject *ret;
+
+    ret = PyObject_CallMethod(file, "close", NULL);
+    if (ret == NULL) {
+        return -1;
+    }
+    Py_DECREF(ret);
+    return 0;
+}
+
diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -171,6 +171,15 @@
         w_array.implementation.shape = []
     return w_array
 
+ at cpython_api([rffi.INT_real], PyObject)
+def _PyArray_DescrFromType(space, typenum):
+    try:
+        dtype = get_dtype_cache(space).dtypes_by_num[typenum]
+        return dtype
+    except KeyError:
+        raise OperationError(space.w_ValueError, space.wrap(
+            '_PyArray_DescrFromType called with invalid dtype %d' % typenum))
+
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, Py_ssize_t], PyObject)
 def _PyArray_FromObject(space, w_obj, typenum, min_depth, max_depth):
     try:
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -265,6 +265,12 @@
                 return obj2;
                 '''
                 ),
+                ("test_DescrFromType", "METH_O",
+                """
+                    Signed typenum = PyInt_AsLong(args);
+                    return _PyArray_DescrFromType(typenum);
+                """
+                ),
                 ], prologue='#include <numpy/arrayobject.h>')
         arr = mod.test_simplenew()
         assert arr.shape == (2, 3)
@@ -278,3 +284,5 @@
         #Make sure these work without errors
         arr = mod.test_FromAny()
         arr = mod.test_FromObject()
+        dt = mod.test_DescrFromType(11)
+        assert dt.num == 11

From noreply at buildbot.pypy.org  Sun Oct 13 07:03:14 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 13 Oct 2013 07:03:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: Kill duplicate method (it is
	implemented below again)
Message-ID: <20131013050314.A8A471C01E3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67339:3fb1e5e6f0b8
Date: 2013-10-13 07:02 +0200
http://bitbucket.org/pypy/pypy/changeset/3fb1e5e6f0b8/

Log:	Kill duplicate method (it is implemented below again)

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -290,9 +290,6 @@
         self.space = space
         self.mmap = mmap
 
-    def get_raw_address(self):
-        return self.mmap.data
-
     def getlength(self):
         return self.mmap.size
 

From noreply at buildbot.pypy.org  Sun Oct 13 07:21:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 13 Oct 2013 07:21:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: Support (without crashing obscurely)
 dictionaries with keys that
Message-ID: <20131013052126.C456D1C01E3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67340:23d01e38f49e
Date: 2013-10-13 07:20 +0200
http://bitbucket.org/pypy/pypy/changeset/23d01e38f49e/

Log:	Support (without crashing obscurely) dictionaries with keys that are
	shorter integer types from rffi.

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -162,6 +162,9 @@
                 fasthashfn = None
             else:
                 fasthashfn = self.key_repr.get_ll_fasthash_function()
+                if getattr(self.key_repr.get_ll_eq_function(),
+                           'no_direct_compare', False):
+                    entrymeths['no_direct_compare'] = True
             if fasthashfn is None:
                 entryfields.append(("f_hash", lltype.Signed))
                 entrymeths['hash'] = ll_hash_from_cache
diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -251,14 +251,15 @@
         raise TyperError("not an integer: %r" % (value,))
 
     def get_ll_eq_function(self):
+        if self._opprefix is None:
+            return ll_eq_shortint
         return None
-    get_ll_gt_function = get_ll_eq_function
-    get_ll_lt_function = get_ll_eq_function
-    get_ll_ge_function = get_ll_eq_function
-    get_ll_le_function = get_ll_eq_function
 
     def get_ll_ge_function(self):
         return None
+    get_ll_gt_function = get_ll_ge_function
+    get_ll_lt_function = get_ll_ge_function
+    get_ll_le_function = get_ll_ge_function
 
     def get_ll_hash_function(self):
         if (sys.maxint == 2147483647 and
@@ -390,6 +391,10 @@
 def ll_hash_long_long(n):
     return intmask(intmask(n) + 9 * intmask(n >> 32))
 
+def ll_eq_shortint(n, m):
+    return intmask(n) == intmask(m)
+ll_eq_shortint.no_direct_compare = True
+
 def ll_check_chr(n):
     if 0 <= n <= 255:
         return
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -999,6 +999,16 @@
         res = f()
         assert res == 1
 
+    def test_dict_with_SHORT_keys(self):
+        def func(x):
+            d = {}
+            d[rffi.cast(rffi.SHORT, 42)] = 123
+            d[rffi.cast(rffi.SHORT, -43)] = 321
+            return d[rffi.cast(rffi.SHORT, x)]
+
+        assert self.interpret(func, [42]) == 123
+        assert self.interpret(func, [2**16 - 43]) == 321
+
     def test_nonnull_hint(self):
         def eq(a, b):
             return a == b

From noreply at buildbot.pypy.org  Sun Oct 13 09:12:53 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 13 Oct 2013 09:12:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix
Message-ID: <20131013071253.E6D361C01F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67341:e99ecdbbbcc7
Date: 2013-10-13 09:12 +0200
http://bitbucket.org/pypy/pypy/changeset/e99ecdbbbcc7/

Log:	Test and fix

diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -251,7 +251,7 @@
         raise TyperError("not an integer: %r" % (value,))
 
     def get_ll_eq_function(self):
-        if self._opprefix is None:
+        if getattr(self, '_opprefix', '?') is None:
             return ll_eq_shortint
         return None
 
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1009,6 +1009,16 @@
         assert self.interpret(func, [42]) == 123
         assert self.interpret(func, [2**16 - 43]) == 321
 
+    def test_dict_with_bool_keys(self):
+        def func(x):
+            d = {}
+            d[False] = 123
+            d[True] = 321
+            return d[x == 42]
+
+        assert self.interpret(func, [5]) == 123
+        assert self.interpret(func, [42]) == 321
+
     def test_nonnull_hint(self):
         def eq(a, b):
             return a == b

From noreply at buildbot.pypy.org  Sun Oct 13 10:23:57 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 13 Oct 2013 10:23:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix translation
Message-ID: <20131013082357.D71E31C0219@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67342:218eb71b2288
Date: 2013-10-13 11:24 +0300
http://bitbucket.org/pypy/pypy/changeset/218eb71b2288/

Log:	fix translation

diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -171,7 +171,7 @@
         w_array.implementation.shape = []
     return w_array
 
- at cpython_api([rffi.INT_real], PyObject)
+ at cpython_api([Py_ssize_t], PyObject)
 def _PyArray_DescrFromType(space, typenum):
     try:
         dtype = get_dtype_cache(space).dtypes_by_num[typenum]

From noreply at buildbot.pypy.org  Sun Oct 13 16:49:38 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 13 Oct 2013 16:49:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: Handle this case as well
Message-ID: <20131013144938.1A0A81C01F7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67344:5507590c8364
Date: 2013-10-13 16:17 +0200
http://bitbucket.org/pypy/pypy/changeset/5507590c8364/

Log:	Handle this case as well

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -155,6 +155,15 @@
                             else:
                                 cache[dststart + i].clear()
                     return
+                elif argboxes[2] in self.new_boxes:
+                    # Fish the descr out of the effectinfo
+                    cache = self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
+                    if cache is not None:
+                        for idx, cache in cache.iteritems():
+                            for frombox in cache.keys():
+                                if not self.is_unescaped(frombox):
+                                    del cache[frombox]
+                    return
             else:
                 # Only invalidate things that are either escaped or arguments
                 for descr, boxes in self.heap_cache.iteritems():

From noreply at buildbot.pypy.org  Sun Oct 13 16:49:39 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 13 Oct 2013 16:49:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: Put this check case back in
Message-ID: <20131013144939.4765B1C0219@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67345:8ddab3e49465
Date: 2013-10-13 16:20 +0200
http://bitbucket.org/pypy/pypy/changeset/8ddab3e49465/

Log:	Put this check case back in

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -155,7 +155,10 @@
                             else:
                                 cache[dststart + i].clear()
                     return
-                elif argboxes[2] in self.new_boxes:
+                elif (
+                    argboxes[2] in self.new_boxes and
+                    len(effectinfo.write_descrs_arrays) == 1
+                ):
                     # Fish the descr out of the effectinfo
                     cache = self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
                     if cache is not None:

From noreply at buildbot.pypy.org  Sun Oct 13 16:49:36 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 13 Oct 2013 16:49:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: make the frontend heapcache smarter,
 take advantage of usually-constant indices, and propate known values
Message-ID: <20131013144936.CBB181C01E3@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67343:886b28486155
Date: 2013-10-13 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/886b28486155/

Log:	make the frontend heapcache smarter, take advantage of usually-
	constant indices, and propate known values

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -118,20 +118,42 @@
             # A special case for ll_arraycopy, because it is so common, and its
             # effects are so well defined.
             elif effectinfo.oopspecindex == effectinfo.OS_ARRAYCOPY:
-                # The destination box
                 if (
-                    argboxes[2] in self.new_boxes and
+                    isinstance(argboxes[3], ConstInt) and
+                    isinstance(argboxes[4], ConstInt) and
+                    isinstance(argboxes[5], ConstInt) and
                     len(effectinfo.write_descrs_arrays) == 1
                 ):
-                    # Fish the descr out of the effectinfo
-                    cache = self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
-                    if cache is not None:
-                        # XXX: in theory the indices of the copy could be
-                        # looked at
-                        for idx, cache in cache.iteritems():
-                            for frombox in cache.keys():
-                                if not self.is_unescaped(frombox):
-                                    del cache[frombox]
+                    descr = effectinfo.write_descrs_arrays[0]
+                    cache = self.heap_array_cache.get(descr, None)
+                    srcstart = argboxes[3].getint()
+                    dststart = argboxes[4].getint()
+                    length = argboxes[5].getint()
+                    for i in xrange(length):
+                        value = self.getarrayitem(
+                            argboxes[1],
+                            ConstInt(srcstart + i),
+                            descr,
+                        )
+                        if value is not None:
+                            self.setarrayitem(
+                                argboxes[2],
+                                ConstInt(dststart + i),
+                                value,
+                                descr,
+                            )
+                        elif cache is not None:
+                            if argboxes[2] in self.new_boxes:
+                                try:
+                                    idx_cache = cache[dststart + i]
+                                except KeyError:
+                                    pass
+                                else:
+                                    for frombox in idx_cache.keys():
+                                        if not self.is_unescaped(frombox):
+                                            del idx_cache[frombox]
+                            else:
+                                cache[dststart + i].clear()
                     return
             else:
                 # Only invalidate things that are either escaped or arguments
@@ -210,9 +232,9 @@
         return new_d
 
     def getarrayitem(self, box, indexbox, descr):
-        box = self._input_indirection(box)
         if not isinstance(indexbox, ConstInt):
             return
+        box = self._input_indirection(box)
         index = indexbox.getint()
         cache = self.heap_array_cache.get(descr, None)
         if cache:
@@ -221,10 +243,10 @@
                 return self._output_indirection(indexcache.get(box, None))
 
     def getarrayitem_now_known(self, box, indexbox, valuebox, descr):
+        if not isinstance(indexbox, ConstInt):
+            return
         box = self._input_indirection(box)
         valuebox = self._input_indirection(valuebox)
-        if not isinstance(indexbox, ConstInt):
-            return
         index = indexbox.getint()
         cache = self.heap_array_cache.setdefault(descr, {})
         indexcache = cache.get(index, None)
diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -372,22 +372,22 @@
         h.invalidate_caches(
             rop.CALL,
             FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
-            [None, None, box2, None, None]
+            [None, box5, box2, index1, index1, index1]
         )
         assert h.getarrayitem(box1, index1, descr1) is box2
         h.invalidate_caches(
             rop.CALL,
             FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
-            [None, None, box3, None, None]
+            [None, box5, box3, index1, index1, index1]
         )
-        assert h.getarrayitem(box1, index1, descr1) is None
+        assert h.getarrayitem(box1, index1, descr1) is box2
 
         h.setarrayitem(box4, index1, box2, descr1)
         assert h.getarrayitem(box4, index1, descr1) is box2
         h.invalidate_caches(
             rop.CALL,
             FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
-            [None, None, box2, None, None]
+            [None, box3, box5, index1, index1, index2]
         )
         assert h.getarrayitem(box4, index1, descr1) is None
 
@@ -399,10 +399,29 @@
         h.invalidate_caches(
             rop.CALL,
             FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr2]),
-            [None, None, box2, None, None]
+            [None, box3, box2, index1, index1, index2]
         )
         assert h.getarrayitem(box1, index1, descr1) is box2
 
+    def test_ll_arraycopy_result_propogated(self):
+        h = HeapCache()
+        h.setarrayitem(box1, index1, box2, descr1)
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            [None, box1, box3, index1, index1, index2]
+        )
+        assert h.getarrayitem(box3, index1, descr1) is box2
+
+    def test_ll_arraycopy_dest_new(self):
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        h.setarrayitem(box3, index1, box4, descr1)
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            [None, box2, box1, index1, index1, index2]
+        )
 
     def test_unescaped(self):
         h = HeapCache()

From noreply at buildbot.pypy.org  Sun Oct 13 16:49:40 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 13 Oct 2013 16:49:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged from upstream
Message-ID: <20131013144940.C501B1C08FE@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67346:d7d87d2a64ba
Date: 2013-10-13 16:41 +0200
http://bitbucket.org/pypy/pypy/changeset/d7d87d2a64ba/

Log:	merged from upstream

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -90,7 +90,8 @@
 .. branch: no-release-gil
 .. branch: safe-win-mmap
 .. branch: boolean-indexing-cleanup
-.. branch: cpyyest-best_base
+.. branch: cpyext-best_base
+.. branch: fileops2
 
 .. branch: nobold-backtrace
 Work on improving UnionError messages and stack trace displays.
diff --git a/pypy/module/cpyext/include/numpy/npy_3kcompat.h b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
--- a/pypy/module/cpyext/include/numpy/npy_3kcompat.h
+++ b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
@@ -0,0 +1,39 @@
+/*
+ * In numpy this is a convenience header file providing compatibility utilities
+ * for supporting Python 2 and Python 3 in the same code base.
+ *
+ * PyPy uses it as a convenient place to add compatability declarations
+ */
+
+#ifndef _NPY_3KCOMPAT_H_
+#define _NPY_3KCOMPAT_H_
+
+#include <numpy/npy_common.h>
+
+#define npy_PyFile_Dup(file, mode) (NULL)
+#define npy_PyFile_DupClose(file, handle) (0)
+
+static NPY_INLINE PyObject*
+npy_PyFile_OpenFile(PyObject *filename, const char *mode)
+{
+    PyObject *open;
+    open = PyDict_GetItemString(PyEval_GetBuiltins(), "open");
+    if (open == NULL) {
+        return NULL;
+    }
+    return PyObject_CallFunction(open, "Os", filename, mode);
+}
+
+static NPY_INLINE int
+npy_PyFile_CloseFile(PyObject *file)
+{
+    PyObject *ret;
+
+    ret = PyObject_CallMethod(file, "close", NULL);
+    if (ret == NULL) {
+        return -1;
+    }
+    Py_DECREF(ret);
+    return 0;
+}
+
diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -171,6 +171,15 @@
         w_array.implementation.shape = []
     return w_array
 
+ at cpython_api([Py_ssize_t], PyObject)
+def _PyArray_DescrFromType(space, typenum):
+    try:
+        dtype = get_dtype_cache(space).dtypes_by_num[typenum]
+        return dtype
+    except KeyError:
+        raise OperationError(space.w_ValueError, space.wrap(
+            '_PyArray_DescrFromType called with invalid dtype %d' % typenum))
+
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, Py_ssize_t], PyObject)
 def _PyArray_FromObject(space, w_obj, typenum, min_depth, max_depth):
     try:
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -265,6 +265,12 @@
                 return obj2;
                 '''
                 ),
+                ("test_DescrFromType", "METH_O",
+                """
+                    Signed typenum = PyInt_AsLong(args);
+                    return _PyArray_DescrFromType(typenum);
+                """
+                ),
                 ], prologue='#include <numpy/arrayobject.h>')
         arr = mod.test_simplenew()
         assert arr.shape == (2, 3)
@@ -278,3 +284,5 @@
         #Make sure these work without errors
         arr = mod.test_FromAny()
         arr = mod.test_FromObject()
+        dt = mod.test_DescrFromType(11)
+        assert dt.num == 11
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -290,9 +290,6 @@
         self.space = space
         self.mmap = mmap
 
-    def get_raw_address(self):
-        return self.mmap.data
-
     def getlength(self):
         return self.mmap.size
 
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1201,7 +1201,6 @@
         # ^^^ a fast path of write-barrier
         #
         if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
-            assert self.card_page_indices > 0
             #
             if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
                 # The source object may have random young pointers.
@@ -1236,6 +1235,7 @@
 
     def manually_copy_card_bits(self, source_addr, dest_addr, length):
         # manually copy the individual card marks from source to dest
+        assert self.card_page_indices > 0
         bytes = self.card_marking_bytes_for_length(length)
         #
         anybyte = 0
diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py
--- a/rpython/rlib/rdtoa.py
+++ b/rpython/rlib/rdtoa.py
@@ -217,13 +217,13 @@
 
         if exp >= 0:
             exp_str = str(exp)
-            if len(exp_str) < 2:
+            if len(exp_str) < 2 and not (flags & rfloat.DTSF_CUT_EXP_0):
                 s += e + '+0' + exp_str
             else:
                 s += e + '+' + exp_str
         else:
             exp_str = str(-exp)
-            if len(exp_str) < 2:
+            if len(exp_str) < 2 and not (flags & rfloat.DTSF_CUT_EXP_0):
                 s += e + '-0' + exp_str
             else:
                 s += e + '-' + exp_str
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -69,6 +69,7 @@
 DTSF_SIGN      = 0x1
 DTSF_ADD_DOT_0 = 0x2
 DTSF_ALT       = 0x4
+DTSF_CUT_EXP_0 = 0x8
 
 DIST_FINITE   = 1
 DIST_NAN      = 2
diff --git a/rpython/rlib/test/test_rdtoa.py b/rpython/rlib/test/test_rdtoa.py
--- a/rpython/rlib/test/test_rdtoa.py
+++ b/rpython/rlib/test/test_rdtoa.py
@@ -29,3 +29,7 @@
 def test_dtoa_precision():
     assert dtoa(1.1, code='f', precision=2) == "1.10"
     assert dtoa(1e12, code='g', precision=12) == "1e+12"
+
+def test_flag_cut_exp_0():
+    assert dtoa(1.1e9, code="g", precision=2, flags=rfloat.DTSF_CUT_EXP_0) == "1.1e+9"
+    assert dtoa(1.1e-9, code="g", precision=2, flags=rfloat.DTSF_CUT_EXP_0) == "1.1e-9"
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -162,6 +162,9 @@
                 fasthashfn = None
             else:
                 fasthashfn = self.key_repr.get_ll_fasthash_function()
+                if getattr(self.key_repr.get_ll_eq_function(),
+                           'no_direct_compare', False):
+                    entrymeths['no_direct_compare'] = True
             if fasthashfn is None:
                 entryfields.append(("f_hash", lltype.Signed))
                 entrymeths['hash'] = ll_hash_from_cache
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -577,9 +577,7 @@
             return -1
 
         m = len(s2.chars)
-        if m == 0:
-            return start
-        elif m == 1:
+        if m == 1:
             return cls.ll_find_char(s1, s2.chars[0], start, end)
 
         return cls.ll_search(s1, s2, start, end, FAST_FIND)
@@ -594,9 +592,7 @@
             return -1
 
         m = len(s2.chars)
-        if m == 0:
-            return end
-        elif m == 1:
+        if m == 1:
             return cls.ll_rfind_char(s1, s2.chars[0], start, end)
 
         return cls.ll_search(s1, s2, start, end, FAST_RFIND)
@@ -611,9 +607,7 @@
             return 0
 
         m = len(s2.chars)
-        if m == 0:
-            return end - start + 1
-        elif m == 1:
+        if m == 1:
             return cls.ll_count_char(s1, s2.chars[0], start, end)
 
         res = cls.ll_search(s1, s2, start, end, FAST_COUNT)
@@ -629,6 +623,14 @@
         n = end - start
         m = len(s2.chars)
 
+        if m == 0:
+            if mode == FAST_COUNT:
+                return end - start + 1
+            elif mode == FAST_RFIND:
+                return end
+            else:
+                return start
+
         w = n - m
 
         if w < 0:
diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -251,14 +251,15 @@
         raise TyperError("not an integer: %r" % (value,))
 
     def get_ll_eq_function(self):
+        if getattr(self, '_opprefix', '?') is None:
+            return ll_eq_shortint
         return None
-    get_ll_gt_function = get_ll_eq_function
-    get_ll_lt_function = get_ll_eq_function
-    get_ll_ge_function = get_ll_eq_function
-    get_ll_le_function = get_ll_eq_function
 
     def get_ll_ge_function(self):
         return None
+    get_ll_gt_function = get_ll_ge_function
+    get_ll_lt_function = get_ll_ge_function
+    get_ll_le_function = get_ll_ge_function
 
     def get_ll_hash_function(self):
         if (sys.maxint == 2147483647 and
@@ -390,6 +391,10 @@
 def ll_hash_long_long(n):
     return intmask(intmask(n) + 9 * intmask(n >> 32))
 
+def ll_eq_shortint(n, m):
+    return intmask(n) == intmask(m)
+ll_eq_shortint.no_direct_compare = True
+
 def ll_check_chr(n):
     if 0 <= n <= 255:
         return
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -999,6 +999,26 @@
         res = f()
         assert res == 1
 
+    def test_dict_with_SHORT_keys(self):
+        def func(x):
+            d = {}
+            d[rffi.cast(rffi.SHORT, 42)] = 123
+            d[rffi.cast(rffi.SHORT, -43)] = 321
+            return d[rffi.cast(rffi.SHORT, x)]
+
+        assert self.interpret(func, [42]) == 123
+        assert self.interpret(func, [2**16 - 43]) == 321
+
+    def test_dict_with_bool_keys(self):
+        def func(x):
+            d = {}
+            d[False] = 123
+            d[True] = 321
+            return d[x == 42]
+
+        assert self.interpret(func, [5]) == 123
+        assert self.interpret(func, [42]) == 321
+
     def test_nonnull_hint(self):
         def eq(a, b):
             return a == b
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -3,6 +3,7 @@
 import py
 
 from rpython.flowspace.model import summary
+from rpython.annotator.model import AnnotatorError
 from rpython.rtyper.lltypesystem.lltype import typeOf, Signed, malloc
 from rpython.rtyper.lltypesystem.rstr import LLHelpers, STR
 from rpython.rtyper.rstr import AbstractLLHelpers
@@ -361,16 +362,16 @@
             res = self.interpret(fn, [i, j])
             assert res == fn(i, j)
 
-    def test_find_TyperError(self):
+    def test_find_AnnotatorError(self):
         const = self.const
         def f():
             s = const('abc')
             s.find(s, 0, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
         def f():
             s = const('abc')
             s.find(s, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
 
     def test_find_empty_string(self):
         const = self.const
@@ -420,9 +421,8 @@
         const = self.const
         def f(i):
             return const("abc").rfind(const(''), i)
-        e = py.test.raises(TyperError, self.interpret, f, [-5])
-        assert str(e.value).startswith(
-            'str.rfind() start must be proven non-negative')
+        e = py.test.raises(AnnotatorError, self.interpret, f, [-5])
+        assert "rfind: not proven to have non-negative start" in str(e.value)
 
     def test_find_char(self):
         const = self.const
@@ -900,16 +900,16 @@
         res = self.interpret(fn, [])
         assert res == 1
 
-    def test_count_TyperError(self):
+    def test_count_AnnotatorError(self):
         const = self.const
         def f():
             s = const('abc')
             s.count(s, 0, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
         def f():
             s = const('abc')
             s.count(s, -10)
-        py.test.raises(TyperError, self.interpret, f, ())
+        py.test.raises(AnnotatorError, self.interpret, f, ())
 
     def test_getitem_exc(self):
         const = self.const

From noreply at buildbot.pypy.org  Sun Oct 13 21:49:36 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 13 Oct 2013 21:49:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: ARRAYCOPY with constant starts and
 constant length doesn't escape its argument
Message-ID: <20131013194936.AF1671C01F7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67347:d1a0c07b6586
Date: 2013-10-13 21:48 +0200
http://bitbucket.org/pypy/pypy/changeset/d1a0c07b6586/

Log:	ARRAYCOPY with constant starts and constant length doesn't escape
	its argument

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -51,10 +51,10 @@
         return self.output_indirections.get(box, box)
 
     def invalidate_caches(self, opnum, descr, argboxes):
-        self.mark_escaped(opnum, argboxes)
+        self.mark_escaped(opnum, descr, argboxes)
         self.clear_caches(opnum, descr, argboxes)
 
-    def mark_escaped(self, opnum, argboxes):
+    def mark_escaped(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
             box, valuebox = argboxes
@@ -69,6 +69,15 @@
                 self.dependencies.setdefault(box, []).append(valuebox)
             else:
                 self._escape(valuebox)
+        elif (opnum == rop.CALL and
+              descr.get_extra_info().oopspecindex == descr.get_extra_info().OS_ARRAYCOPY and
+              isinstance(argboxes[3], ConstInt) and
+              isinstance(argboxes[4], ConstInt) and
+              isinstance(argboxes[5], ConstInt) and
+              len(descr.get_extra_info().write_descrs_arrays) == 1):
+            # ARRAYCOPY with constant starts and constant length doesn't escape
+            # its argument
+            pass
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
         elif (opnum != rop.GETFIELD_GC and
diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -1,6 +1,6 @@
 from rpython.jit.metainterp.heapcache import HeapCache
 from rpython.jit.metainterp.resoperation import rop
-from rpython.jit.metainterp.history import ConstInt
+from rpython.jit.metainterp.history import ConstInt, BoxInt
 
 box1 = "box1"
 box2 = "box2"
@@ -73,7 +73,6 @@
         assert not h.is_nonstandard_virtualizable(1)
         assert not h.is_nonstandard_virtualizable(2)
 
-
     def test_heapcache_fields(self):
         h = HeapCache()
         assert h.getfield(box1, descr1) is None
@@ -278,7 +277,6 @@
         assert h.getarrayitem(box1, index1, descr1) is None
         assert h.getarrayitem(box1, index2, descr1) is None
 
-
     def test_replace_box(self):
         h = HeapCache()
         h.setfield(box1, box2, descr1)
@@ -423,6 +421,25 @@
             [None, box2, box1, index1, index1, index2]
         )
 
+    def test_ll_arraycopy_doesnt_escape_arrays(self):
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        h.new_array(box2, lengthbox2)
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            [None, box2, box1, index1, index1, index2]
+        )
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            [None, box2, box1, index1, index1, BoxInt()]
+        )
+        assert not h.is_unescaped(box1)
+        assert not h.is_unescaped(box2)
+
     def test_unescaped(self):
         h = HeapCache()
         assert not h.is_unescaped(box1)

From noreply at buildbot.pypy.org  Sun Oct 13 22:54:34 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 13 Oct 2013 22:54:34 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Done
Message-ID: <20131013205434.362701C01E3@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5069:37671f5ab693
Date: 2013-10-13 22:54 +0200
http://bitbucket.org/pypy/extradoc/changeset/37671f5ab693/

Log:	Done

diff --git a/planning/jit.txt b/planning/jit.txt
--- a/planning/jit.txt
+++ b/planning/jit.txt
@@ -45,9 +45,6 @@
   (SETINTERIORFIELD, GETINTERIORFIELD). This is needed for the previous item to
   fully work.
 
-- {}.update({}) is not fully unrolled and constant folded because HeapCache
-  loses track of values in virtual-to-virtual ARRAY_COPY calls.
-
 - ovfcheck(a << b) will do ``result >> b`` and check that the result is equal
   to ``a``, instead of looking at the x86 flags.
 

From noreply at buildbot.pypy.org  Mon Oct 14 05:00:59 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 14 Oct 2013 05:00:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: document how the file works
Message-ID: <20131014030059.98F471C01E3@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67348:61192c252609
Date: 2013-10-13 02:06 +0300
http://bitbucket.org/pypy/pypy/changeset/61192c252609/

Log:	document how the file works

diff --git a/pypy/module/cpyext/include/numpy/npy_3kcompat.h b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
--- a/pypy/module/cpyext/include/numpy/npy_3kcompat.h
+++ b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
@@ -3,6 +3,8 @@
  * for supporting Python 2 and Python 3 in the same code base.
  *
  * PyPy uses it as a convenient place to add compatability declarations
+ * It will be copied by numpy/core/setup.py by install_data to
+ * site-packages/numpy/core/includes/numpy 
  */
 
 #ifndef _NPY_3KCOMPAT_H_

From noreply at buildbot.pypy.org  Mon Oct 14 05:01:00 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 14 Oct 2013 05:01:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: numpy compatible capi fixes
Message-ID: <20131014030100.C6BE01C01E3@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67349:a3f27dccf08a
Date: 2013-10-13 22:16 +0300
http://bitbucket.org/pypy/pypy/changeset/a3f27dccf08a/

Log:	numpy compatible capi fixes

diff --git a/pypy/module/cpyext/include/numpy/arrayobject.h b/pypy/module/cpyext/include/numpy/arrayobject.h
--- a/pypy/module/cpyext/include/numpy/arrayobject.h
+++ b/pypy/module/cpyext/include/numpy/arrayobject.h
@@ -1,5 +1,8 @@
 
-/* NDArray object interface - S. H. Muller, 2013/07/26 */
+/* NDArray object interface - S. H. Muller, 2013/07/26 
+ * It will be copied by numpy/core/setup.py by install_data to
+ * site-packages/numpy/core/includes/numpy  
+*/
 
 #ifndef Py_NDARRAYOBJECT_H
 #define Py_NDARRAYOBJECT_H
@@ -9,7 +12,6 @@
 
 #include "old_defines.h"
 
-#define NPY_INLINE
 #define NPY_UNUSED(x) x
 #define PyArray_MAX(a,b) (((a)>(b))?(a):(b))
 #define PyArray_MIN(a,b) (((a)<(b))?(a):(b))
@@ -22,11 +24,12 @@
 
 typedef unsigned char npy_bool;
 typedef unsigned char npy_uint8;
+typedef unsigned short npy_uint16;
+typedef signed short npy_int16;
+typedef signed char npy_int8;
 typedef int npy_int;
 
-#ifndef npy_intp
-#define npy_intp long
-#endif
+typedef long npy_intp;
 #ifndef NPY_INTP_FMT
 #define NPY_INTP_FMT "ld"
 #endif
diff --git a/pypy/module/cpyext/include/numpy/npy_3kcompat.h b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
--- a/pypy/module/cpyext/include/numpy/npy_3kcompat.h
+++ b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
@@ -4,8 +4,8 @@
  *
  * PyPy uses it as a convenient place to add compatability declarations
  * It will be copied by numpy/core/setup.py by install_data to
- * site-packages/numpy/core/includes/numpy 
- */
+ * site-packages/numpy/core/includes/numpy  
+*/
 
 #ifndef _NPY_3KCOMPAT_H_
 #define _NPY_3KCOMPAT_H_
@@ -38,4 +38,4 @@
     Py_DECREF(ret);
     return 0;
 }
-
+#endif

From noreply at buildbot.pypy.org  Mon Oct 14 05:01:02 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 14 Oct 2013 05:01:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: allow comparison between ndarray and
	string
Message-ID: <20131014030102.175F31C01E3@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67350:413f957cdd2b
Date: 2013-10-13 23:53 +0300
http://bitbucket.org/pypy/pypy/changeset/413f957cdd2b/

Log:	allow comparison between ndarray and string

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -954,6 +954,13 @@
     def descr___array_finalize__(self, space, w_obj):
         pass
 
+    def descr___array_wrap__(self, space, w_obj, w_context=None):
+        return w_obj
+
+    def descr___array_prepare__(self, space, w_obj, w_context=None):
+        return w_obj
+        pass
+
 @unwrap_spec(offset=int, order=str)
 def descr_new_array(space, w_subtype, w_shape, w_dtype=None, w_buffer=None,
                     offset=0, w_strides=None, order='C'):
@@ -1144,7 +1151,8 @@
     __reduce__ = interp2app(W_NDimArray.descr_reduce),
     __setstate__ = interp2app(W_NDimArray.descr_setstate),
     __array_finalize__ = interp2app(W_NDimArray.descr___array_finalize__),
-
+    __array_prepare__ = interp2app(W_NDimArray.descr___array_prepare__),
+    __array_wrap__ = interp2app(W_NDimArray.descr___array_wrap__),
     __array__         = interp2app(W_NDimArray.descr___array__),
 )
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -342,6 +342,9 @@
         if w_ldtype.is_str_type() and w_rdtype.is_str_type() and \
            self.comparison_func:
             pass
+        elif (w_ldtype.is_str_type() or w_rdtype.is_str_type()) and \
+            self.comparison_func and w_out is None:
+            return space.wrap(False)
         elif (w_ldtype.is_flexible_type() or \
                 w_rdtype.is_flexible_type()):
             raise OperationError(space.w_TypeError, space.wrap(
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -712,7 +712,8 @@
 
     def test_comparisons(self):
         import operator
-        from numpypy import equal, not_equal, less, less_equal, greater, greater_equal
+        from numpypy import (equal, not_equal, less, less_equal, greater,
+                            greater_equal, arange)
 
         for ufunc, func in [
             (equal, operator.eq),
@@ -735,7 +736,9 @@
                 (3, 3.5),
             ]:
                 assert ufunc(a, b) == func(a, b)
-
+        c = arange(10)
+        val = c == 'abcdefg'
+        assert val == False
 
     def test_count_nonzero(self):
         from numpypy import count_nonzero

From noreply at buildbot.pypy.org  Mon Oct 14 12:57:21 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 12:57:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: a tool for analysiing gc pause
	histograms
Message-ID: <20131014105721.EEF6E1C02E2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67351:4f6d0910df19
Date: 2013-10-14 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/4f6d0910df19/

Log:	a tool for analysiing gc pause histograms

diff --git a/rpython/tool/gcanalyze.py b/rpython/tool/gcanalyze.py
new file mode 100755
--- /dev/null
+++ b/rpython/tool/gcanalyze.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+""" Parse gcdumps. Use by saying PYPYLOG=gc-collect:log pypy <program>
+and run it by:
+
+gcanalyze.py logfile [--plot]
+"""
+
+import sys
+from rpython.tool.logparser import parse_log
+
+NO_BUCKETS = 8
+
+def main(arg):
+    log = parse_log(open(arg).readlines())
+    all = []
+    for entry in log:
+        if entry[0].startswith('gc-collect'):
+            start = entry[1]
+            end = entry[2]
+            all.append(float(end - start) / 1000000)
+    avg = sum(all) / len(all)
+    max_t = max(all)
+    print "AVG:", "%.1fms" % avg, "MAX:", "%.1fms" % max_t
+    buckets = [0] * (NO_BUCKETS + 1)
+    for item in all:
+        bucket = int(item / max_t * NO_BUCKETS)
+        buckets[bucket] += 1
+    l1 = ["%.1fms" % ((i + 1) * max_t / NO_BUCKETS) for i in range(NO_BUCKETS)]
+    l2 = [str(i) for i in buckets[1:]]
+    for i, elem in enumerate(l1):
+        l2[i] += " " * (len(elem) - len(l2[i]))
+    print " ".join(l1)
+    print " ".join(l2)
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2 or len(sys.argv) > 3:
+        print __doc__
+        sys.exit(1)
+    plot = False
+    if len(sys.argv) == 3:
+        if sys.argv[1] == '--plot':
+            plot = True
+            arg = sys.argv[2]
+        elif sys.argv[2] == '--plot':
+            plot = True
+            arg = sys.argv[1]
+        else:
+            print "Wrong command line options:", sys.argv
+            sys.exit(1)
+    else:
+        arg = sys.argv[1]
+    main(arg)

From noreply at buildbot.pypy.org  Mon Oct 14 12:57:24 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 12:57:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131014105724.046861C067F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67352:6f41123c181b
Date: 2013-10-14 12:56 +0200
http://bitbucket.org/pypy/pypy/changeset/6f41123c181b/

Log:	merge

diff --git a/pypy/module/cpyext/include/numpy/arrayobject.h b/pypy/module/cpyext/include/numpy/arrayobject.h
--- a/pypy/module/cpyext/include/numpy/arrayobject.h
+++ b/pypy/module/cpyext/include/numpy/arrayobject.h
@@ -1,5 +1,8 @@
 
-/* NDArray object interface - S. H. Muller, 2013/07/26 */
+/* NDArray object interface - S. H. Muller, 2013/07/26 
+ * It will be copied by numpy/core/setup.py by install_data to
+ * site-packages/numpy/core/includes/numpy  
+*/
 
 #ifndef Py_NDARRAYOBJECT_H
 #define Py_NDARRAYOBJECT_H
@@ -9,7 +12,6 @@
 
 #include "old_defines.h"
 
-#define NPY_INLINE
 #define NPY_UNUSED(x) x
 #define PyArray_MAX(a,b) (((a)>(b))?(a):(b))
 #define PyArray_MIN(a,b) (((a)<(b))?(a):(b))
@@ -22,11 +24,12 @@
 
 typedef unsigned char npy_bool;
 typedef unsigned char npy_uint8;
+typedef unsigned short npy_uint16;
+typedef signed short npy_int16;
+typedef signed char npy_int8;
 typedef int npy_int;
 
-#ifndef npy_intp
-#define npy_intp long
-#endif
+typedef long npy_intp;
 #ifndef NPY_INTP_FMT
 #define NPY_INTP_FMT "ld"
 #endif
diff --git a/pypy/module/cpyext/include/numpy/npy_3kcompat.h b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
--- a/pypy/module/cpyext/include/numpy/npy_3kcompat.h
+++ b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
@@ -0,0 +1,41 @@
+/*
+ * In numpy this is a convenience header file providing compatibility utilities
+ * for supporting Python 2 and Python 3 in the same code base.
+ *
+ * PyPy uses it as a convenient place to add compatability declarations
+ * It will be copied by numpy/core/setup.py by install_data to
+ * site-packages/numpy/core/includes/numpy  
+*/
+
+#ifndef _NPY_3KCOMPAT_H_
+#define _NPY_3KCOMPAT_H_
+
+#include <numpy/npy_common.h>
+
+#define npy_PyFile_Dup(file, mode) (NULL)
+#define npy_PyFile_DupClose(file, handle) (0)
+
+static NPY_INLINE PyObject*
+npy_PyFile_OpenFile(PyObject *filename, const char *mode)
+{
+    PyObject *open;
+    open = PyDict_GetItemString(PyEval_GetBuiltins(), "open");
+    if (open == NULL) {
+        return NULL;
+    }
+    return PyObject_CallFunction(open, "Os", filename, mode);
+}
+
+static NPY_INLINE int
+npy_PyFile_CloseFile(PyObject *file)
+{
+    PyObject *ret;
+
+    ret = PyObject_CallMethod(file, "close", NULL);
+    if (ret == NULL) {
+        return -1;
+    }
+    Py_DECREF(ret);
+    return 0;
+}
+#endif
diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -171,6 +171,15 @@
         w_array.implementation.shape = []
     return w_array
 
+ at cpython_api([Py_ssize_t], PyObject)
+def _PyArray_DescrFromType(space, typenum):
+    try:
+        dtype = get_dtype_cache(space).dtypes_by_num[typenum]
+        return dtype
+    except KeyError:
+        raise OperationError(space.w_ValueError, space.wrap(
+            '_PyArray_DescrFromType called with invalid dtype %d' % typenum))
+
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, Py_ssize_t], PyObject)
 def _PyArray_FromObject(space, w_obj, typenum, min_depth, max_depth):
     try:
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -265,6 +265,12 @@
                 return obj2;
                 '''
                 ),
+                ("test_DescrFromType", "METH_O",
+                """
+                    Signed typenum = PyInt_AsLong(args);
+                    return _PyArray_DescrFromType(typenum);
+                """
+                ),
                 ], prologue='#include <numpy/arrayobject.h>')
         arr = mod.test_simplenew()
         assert arr.shape == (2, 3)
@@ -278,3 +284,5 @@
         #Make sure these work without errors
         arr = mod.test_FromAny()
         arr = mod.test_FromObject()
+        dt = mod.test_DescrFromType(11)
+        assert dt.num == 11
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -954,6 +954,13 @@
     def descr___array_finalize__(self, space, w_obj):
         pass
 
+    def descr___array_wrap__(self, space, w_obj, w_context=None):
+        return w_obj
+
+    def descr___array_prepare__(self, space, w_obj, w_context=None):
+        return w_obj
+        pass
+
 @unwrap_spec(offset=int, order=str)
 def descr_new_array(space, w_subtype, w_shape, w_dtype=None, w_buffer=None,
                     offset=0, w_strides=None, order='C'):
@@ -1144,7 +1151,8 @@
     __reduce__ = interp2app(W_NDimArray.descr_reduce),
     __setstate__ = interp2app(W_NDimArray.descr_setstate),
     __array_finalize__ = interp2app(W_NDimArray.descr___array_finalize__),
-
+    __array_prepare__ = interp2app(W_NDimArray.descr___array_prepare__),
+    __array_wrap__ = interp2app(W_NDimArray.descr___array_wrap__),
     __array__         = interp2app(W_NDimArray.descr___array__),
 )
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -342,6 +342,9 @@
         if w_ldtype.is_str_type() and w_rdtype.is_str_type() and \
            self.comparison_func:
             pass
+        elif (w_ldtype.is_str_type() or w_rdtype.is_str_type()) and \
+            self.comparison_func and w_out is None:
+            return space.wrap(False)
         elif (w_ldtype.is_flexible_type() or \
                 w_rdtype.is_flexible_type()):
             raise OperationError(space.w_TypeError, space.wrap(
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -712,7 +712,8 @@
 
     def test_comparisons(self):
         import operator
-        from numpypy import equal, not_equal, less, less_equal, greater, greater_equal
+        from numpypy import (equal, not_equal, less, less_equal, greater,
+                            greater_equal, arange)
 
         for ufunc, func in [
             (equal, operator.eq),
@@ -735,7 +736,9 @@
                 (3, 3.5),
             ]:
                 assert ufunc(a, b) == func(a, b)
-
+        c = arange(10)
+        val = c == 'abcdefg'
+        assert val == False
 
     def test_count_nonzero(self):
         from numpypy import count_nonzero
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -290,9 +290,6 @@
         self.space = space
         self.mmap = mmap
 
-    def get_raw_address(self):
-        return self.mmap.data
-
     def getlength(self):
         return self.mmap.size
 
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -51,10 +51,10 @@
         return self.output_indirections.get(box, box)
 
     def invalidate_caches(self, opnum, descr, argboxes):
-        self.mark_escaped(opnum, argboxes)
+        self.mark_escaped(opnum, descr, argboxes)
         self.clear_caches(opnum, descr, argboxes)
 
-    def mark_escaped(self, opnum, argboxes):
+    def mark_escaped(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
             box, valuebox = argboxes
@@ -69,6 +69,15 @@
                 self.dependencies.setdefault(box, []).append(valuebox)
             else:
                 self._escape(valuebox)
+        elif (opnum == rop.CALL and
+              descr.get_extra_info().oopspecindex == descr.get_extra_info().OS_ARRAYCOPY and
+              isinstance(argboxes[3], ConstInt) and
+              isinstance(argboxes[4], ConstInt) and
+              isinstance(argboxes[5], ConstInt) and
+              len(descr.get_extra_info().write_descrs_arrays) == 1):
+            # ARRAYCOPY with constant starts and constant length doesn't escape
+            # its argument
+            pass
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
         elif (opnum != rop.GETFIELD_GC and
@@ -118,16 +127,50 @@
             # A special case for ll_arraycopy, because it is so common, and its
             # effects are so well defined.
             elif effectinfo.oopspecindex == effectinfo.OS_ARRAYCOPY:
-                # The destination box
                 if (
+                    isinstance(argboxes[3], ConstInt) and
+                    isinstance(argboxes[4], ConstInt) and
+                    isinstance(argboxes[5], ConstInt) and
+                    len(effectinfo.write_descrs_arrays) == 1
+                ):
+                    descr = effectinfo.write_descrs_arrays[0]
+                    cache = self.heap_array_cache.get(descr, None)
+                    srcstart = argboxes[3].getint()
+                    dststart = argboxes[4].getint()
+                    length = argboxes[5].getint()
+                    for i in xrange(length):
+                        value = self.getarrayitem(
+                            argboxes[1],
+                            ConstInt(srcstart + i),
+                            descr,
+                        )
+                        if value is not None:
+                            self.setarrayitem(
+                                argboxes[2],
+                                ConstInt(dststart + i),
+                                value,
+                                descr,
+                            )
+                        elif cache is not None:
+                            if argboxes[2] in self.new_boxes:
+                                try:
+                                    idx_cache = cache[dststart + i]
+                                except KeyError:
+                                    pass
+                                else:
+                                    for frombox in idx_cache.keys():
+                                        if not self.is_unescaped(frombox):
+                                            del idx_cache[frombox]
+                            else:
+                                cache[dststart + i].clear()
+                    return
+                elif (
                     argboxes[2] in self.new_boxes and
                     len(effectinfo.write_descrs_arrays) == 1
                 ):
                     # Fish the descr out of the effectinfo
                     cache = self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
                     if cache is not None:
-                        # XXX: in theory the indices of the copy could be
-                        # looked at
                         for idx, cache in cache.iteritems():
                             for frombox in cache.keys():
                                 if not self.is_unescaped(frombox):
@@ -210,9 +253,9 @@
         return new_d
 
     def getarrayitem(self, box, indexbox, descr):
-        box = self._input_indirection(box)
         if not isinstance(indexbox, ConstInt):
             return
+        box = self._input_indirection(box)
         index = indexbox.getint()
         cache = self.heap_array_cache.get(descr, None)
         if cache:
@@ -221,10 +264,10 @@
                 return self._output_indirection(indexcache.get(box, None))
 
     def getarrayitem_now_known(self, box, indexbox, valuebox, descr):
+        if not isinstance(indexbox, ConstInt):
+            return
         box = self._input_indirection(box)
         valuebox = self._input_indirection(valuebox)
-        if not isinstance(indexbox, ConstInt):
-            return
         index = indexbox.getint()
         cache = self.heap_array_cache.setdefault(descr, {})
         indexcache = cache.get(index, None)
diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -1,6 +1,6 @@
 from rpython.jit.metainterp.heapcache import HeapCache
 from rpython.jit.metainterp.resoperation import rop
-from rpython.jit.metainterp.history import ConstInt
+from rpython.jit.metainterp.history import ConstInt, BoxInt
 
 box1 = "box1"
 box2 = "box2"
@@ -73,7 +73,6 @@
         assert not h.is_nonstandard_virtualizable(1)
         assert not h.is_nonstandard_virtualizable(2)
 
-
     def test_heapcache_fields(self):
         h = HeapCache()
         assert h.getfield(box1, descr1) is None
@@ -278,7 +277,6 @@
         assert h.getarrayitem(box1, index1, descr1) is None
         assert h.getarrayitem(box1, index2, descr1) is None
 
-
     def test_replace_box(self):
         h = HeapCache()
         h.setfield(box1, box2, descr1)
@@ -372,22 +370,22 @@
         h.invalidate_caches(
             rop.CALL,
             FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
-            [None, None, box2, None, None]
+            [None, box5, box2, index1, index1, index1]
         )
         assert h.getarrayitem(box1, index1, descr1) is box2
         h.invalidate_caches(
             rop.CALL,
             FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
-            [None, None, box3, None, None]
+            [None, box5, box3, index1, index1, index1]
         )
-        assert h.getarrayitem(box1, index1, descr1) is None
+        assert h.getarrayitem(box1, index1, descr1) is box2
 
         h.setarrayitem(box4, index1, box2, descr1)
         assert h.getarrayitem(box4, index1, descr1) is box2
         h.invalidate_caches(
             rop.CALL,
             FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
-            [None, None, box2, None, None]
+            [None, box3, box5, index1, index1, index2]
         )
         assert h.getarrayitem(box4, index1, descr1) is None
 
@@ -399,10 +397,48 @@
         h.invalidate_caches(
             rop.CALL,
             FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr2]),
-            [None, None, box2, None, None]
+            [None, box3, box2, index1, index1, index2]
         )
         assert h.getarrayitem(box1, index1, descr1) is box2
 
+    def test_ll_arraycopy_result_propogated(self):
+        h = HeapCache()
+        h.setarrayitem(box1, index1, box2, descr1)
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            [None, box1, box3, index1, index1, index2]
+        )
+        assert h.getarrayitem(box3, index1, descr1) is box2
+
+    def test_ll_arraycopy_dest_new(self):
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        h.setarrayitem(box3, index1, box4, descr1)
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            [None, box2, box1, index1, index1, index2]
+        )
+
+    def test_ll_arraycopy_doesnt_escape_arrays(self):
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        h.new_array(box2, lengthbox2)
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            [None, box2, box1, index1, index1, index2]
+        )
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            [None, box2, box1, index1, index1, BoxInt()]
+        )
+        assert not h.is_unescaped(box1)
+        assert not h.is_unescaped(box2)
 
     def test_unescaped(self):
         h = HeapCache()
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -162,6 +162,9 @@
                 fasthashfn = None
             else:
                 fasthashfn = self.key_repr.get_ll_fasthash_function()
+                if getattr(self.key_repr.get_ll_eq_function(),
+                           'no_direct_compare', False):
+                    entrymeths['no_direct_compare'] = True
             if fasthashfn is None:
                 entryfields.append(("f_hash", lltype.Signed))
                 entrymeths['hash'] = ll_hash_from_cache
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -577,9 +577,7 @@
             return -1
 
         m = len(s2.chars)
-        if m == 0:
-            return start
-        elif m == 1:
+        if m == 1:
             return cls.ll_find_char(s1, s2.chars[0], start, end)
 
         return cls.ll_search(s1, s2, start, end, FAST_FIND)
@@ -594,9 +592,7 @@
             return -1
 
         m = len(s2.chars)
-        if m == 0:
-            return end
-        elif m == 1:
+        if m == 1:
             return cls.ll_rfind_char(s1, s2.chars[0], start, end)
 
         return cls.ll_search(s1, s2, start, end, FAST_RFIND)
@@ -611,9 +607,7 @@
             return 0
 
         m = len(s2.chars)
-        if m == 0:
-            return end - start + 1
-        elif m == 1:
+        if m == 1:
             return cls.ll_count_char(s1, s2.chars[0], start, end)
 
         res = cls.ll_search(s1, s2, start, end, FAST_COUNT)
@@ -629,6 +623,14 @@
         n = end - start
         m = len(s2.chars)
 
+        if m == 0:
+            if mode == FAST_COUNT:
+                return end - start + 1
+            elif mode == FAST_RFIND:
+                return end
+            else:
+                return start
+
         w = n - m
 
         if w < 0:
diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -251,14 +251,15 @@
         raise TyperError("not an integer: %r" % (value,))
 
     def get_ll_eq_function(self):
+        if getattr(self, '_opprefix', '?') is None:
+            return ll_eq_shortint
         return None
-    get_ll_gt_function = get_ll_eq_function
-    get_ll_lt_function = get_ll_eq_function
-    get_ll_ge_function = get_ll_eq_function
-    get_ll_le_function = get_ll_eq_function
 
     def get_ll_ge_function(self):
         return None
+    get_ll_gt_function = get_ll_ge_function
+    get_ll_lt_function = get_ll_ge_function
+    get_ll_le_function = get_ll_ge_function
 
     def get_ll_hash_function(self):
         if (sys.maxint == 2147483647 and
@@ -390,6 +391,10 @@
 def ll_hash_long_long(n):
     return intmask(intmask(n) + 9 * intmask(n >> 32))
 
+def ll_eq_shortint(n, m):
+    return intmask(n) == intmask(m)
+ll_eq_shortint.no_direct_compare = True
+
 def ll_check_chr(n):
     if 0 <= n <= 255:
         return
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -999,6 +999,26 @@
         res = f()
         assert res == 1
 
+    def test_dict_with_SHORT_keys(self):
+        def func(x):
+            d = {}
+            d[rffi.cast(rffi.SHORT, 42)] = 123
+            d[rffi.cast(rffi.SHORT, -43)] = 321
+            return d[rffi.cast(rffi.SHORT, x)]
+
+        assert self.interpret(func, [42]) == 123
+        assert self.interpret(func, [2**16 - 43]) == 321
+
+    def test_dict_with_bool_keys(self):
+        def func(x):
+            d = {}
+            d[False] = 123
+            d[True] = 321
+            return d[x == 42]
+
+        assert self.interpret(func, [5]) == 123
+        assert self.interpret(func, [42]) == 321
+
     def test_nonnull_hint(self):
         def eq(a, b):
             return a == b

From noreply at buildbot.pypy.org  Mon Oct 14 13:12:48 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 13:12:48 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: a minor difference in approach
 to estimating the size of increment
Message-ID: <20131014111248.D6BBC1C02C2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: incremental-gc
Changeset: r67353:417a7117f8d7
Date: 2013-10-14 13:12 +0200
http://bitbucket.org/pypy/pypy/changeset/417a7117f8d7/

Log:	a minor difference in approach to estimating the size of increment

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -10,6 +10,12 @@
                          be smaller than the nursery size and bigger than the
                          biggest object we can allotate in the nursery.
 
+ PYPY_GC_INCREMENT_STEP  The size of memory marked during the marking step.
+                         Default is size of nursery * 2. If you mark it too high
+                         your GC is not incremental at all. The minimum is set
+                         to size that survives minor collection * 1.5 so we
+                         reclaim anything all the time.
+
  PYPY_GC_MAJOR_COLLECT   Major collection memory factor.  Default is '1.82',
                          which means trigger a major collection when the
                          memory consumed equals 1.82 times the memory
@@ -358,6 +364,7 @@
         # allocate the nursery of the final size.
         if not self.read_from_env:
             self.allocate_nursery()
+            self.gc_increment_step = self.nursery_size * 4
         else:
             #
             defaultsize = self.nursery_size
@@ -411,6 +418,12 @@
                 self.max_delta = float(max_delta)
             else:
                 self.max_delta = 0.125 * env.get_total_memory()
+
+            gc_increment_step = env.read_uint_from_env('PYPY_GC_INCREMENT_STEP')
+            if gc_increment_step > 0:
+                self.gc_increment_step = gc_increment_step
+            else:
+                self.gc_increment_step = newsize * 4
             #
             self.minor_collection()    # to empty the nursery
             llarena.arena_free(self.nursery)
@@ -1375,6 +1388,7 @@
         # young objects; only objects directly referenced by roots
         # are copied out or flagged.  They are also added to the list
         # 'old_objects_pointing_to_young'.
+        self.nursery_surviving_size = 0
         self.collect_roots_in_nursery()
         #
         while True:
@@ -1581,6 +1595,7 @@
             # HAS_SHADOW flag either.  We must move it out of the nursery,
             # into a new nonmovable location.
             totalsize = size_gc_header + self.get_size(obj)
+            self.nursery_surviving_size += raw_malloc_usage(totalsize)
             newhdr = self._malloc_out_of_nursery(totalsize)
             #
         elif self.is_forwarded(obj):
@@ -1760,8 +1775,10 @@
         elif self.gc_state == STATE_MARKING:
             debug_print("number of objects to mark",
                         self.objects_to_trace.length())
-
-            estimate = self.nursery_size // 10    # XXX
+            estimate = self.gc_increment_step
+            estimate_from_nursery = self.nursery_surviving_size * 2
+            if estimate_from_nursery > estimate:
+                estimate = estimate_from_nursery
             self.visit_all_objects_step(estimate)
 
             # XXX A simplifying assumption that should be checked,
@@ -1946,13 +1963,15 @@
     def visit_all_objects(self):
         self.visit_all_objects_step(sys.maxint)
 
-    def visit_all_objects_step(self, nobjects):
+    def visit_all_objects_step(self, size_to_track):
         # Objects can be added to pending by visit
         pending = self.objects_to_trace
-        while nobjects > 0 and pending.non_empty():
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        while size_to_track > 0 and pending.non_empty():
             obj = pending.pop()
             self.visit(obj)
-            nobjects -= 1
+            totalsize = size_gc_header + self.get_size(obj)
+            size_to_track -= raw_malloc_usage(totalsize)
 
     def visit(self, obj):
         #

From noreply at buildbot.pypy.org  Mon Oct 14 13:16:24 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 14 Oct 2013 13:16:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: Optimize based on the fact that int_add
	is commutative
Message-ID: <20131014111624.BF0091C02C2@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67354:1b92bf00c279
Date: 2013-10-14 00:22 +0200
http://bitbucket.org/pypy/pypy/changeset/1b92bf00c279/

Log:	Optimize based on the fact that int_add is commutative

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -129,6 +129,7 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
+            self.pure(rop.INT_ADD, [op.getarg(1), op.getarg(0)], op.result)
             # Synthesize the reverse op for optimize_default to reuse
             self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
             self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -3670,6 +3670,20 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_int_add_commutative(self):
+        ops = """
+        [i0, i1]
+        i2 = int_add(i0, i1)
+        i3 = int_add(i1, i0)
+        jump(i2, i3)
+        """
+        expected = """
+        [i0, i1]
+        i2 = int_add(i0, i1)
+        jump(i2, i2)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_framestackdepth_overhead(self):
         ops = """
         [p0, i22]

From noreply at buildbot.pypy.org  Mon Oct 14 13:16:26 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 14 Oct 2013 13:16:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20131014111626.5546D1C02C2@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67355:d57e7620cc8b
Date: 2013-10-14 13:15 +0200
http://bitbucket.org/pypy/pypy/changeset/d57e7620cc8b/

Log:	merged upstream

diff --git a/pypy/module/cpyext/include/numpy/arrayobject.h b/pypy/module/cpyext/include/numpy/arrayobject.h
--- a/pypy/module/cpyext/include/numpy/arrayobject.h
+++ b/pypy/module/cpyext/include/numpy/arrayobject.h
@@ -1,5 +1,8 @@
 
-/* NDArray object interface - S. H. Muller, 2013/07/26 */
+/* NDArray object interface - S. H. Muller, 2013/07/26 
+ * It will be copied by numpy/core/setup.py by install_data to
+ * site-packages/numpy/core/includes/numpy  
+*/
 
 #ifndef Py_NDARRAYOBJECT_H
 #define Py_NDARRAYOBJECT_H
@@ -9,7 +12,6 @@
 
 #include "old_defines.h"
 
-#define NPY_INLINE
 #define NPY_UNUSED(x) x
 #define PyArray_MAX(a,b) (((a)>(b))?(a):(b))
 #define PyArray_MIN(a,b) (((a)<(b))?(a):(b))
@@ -22,11 +24,12 @@
 
 typedef unsigned char npy_bool;
 typedef unsigned char npy_uint8;
+typedef unsigned short npy_uint16;
+typedef signed short npy_int16;
+typedef signed char npy_int8;
 typedef int npy_int;
 
-#ifndef npy_intp
-#define npy_intp long
-#endif
+typedef long npy_intp;
 #ifndef NPY_INTP_FMT
 #define NPY_INTP_FMT "ld"
 #endif
diff --git a/pypy/module/cpyext/include/numpy/npy_3kcompat.h b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
--- a/pypy/module/cpyext/include/numpy/npy_3kcompat.h
+++ b/pypy/module/cpyext/include/numpy/npy_3kcompat.h
@@ -3,7 +3,9 @@
  * for supporting Python 2 and Python 3 in the same code base.
  *
  * PyPy uses it as a convenient place to add compatability declarations
- */
+ * It will be copied by numpy/core/setup.py by install_data to
+ * site-packages/numpy/core/includes/numpy  
+*/
 
 #ifndef _NPY_3KCOMPAT_H_
 #define _NPY_3KCOMPAT_H_
@@ -36,4 +38,4 @@
     Py_DECREF(ret);
     return 0;
 }
-
+#endif
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -954,6 +954,13 @@
     def descr___array_finalize__(self, space, w_obj):
         pass
 
+    def descr___array_wrap__(self, space, w_obj, w_context=None):
+        return w_obj
+
+    def descr___array_prepare__(self, space, w_obj, w_context=None):
+        return w_obj
+        pass
+
 @unwrap_spec(offset=int, order=str)
 def descr_new_array(space, w_subtype, w_shape, w_dtype=None, w_buffer=None,
                     offset=0, w_strides=None, order='C'):
@@ -1144,7 +1151,8 @@
     __reduce__ = interp2app(W_NDimArray.descr_reduce),
     __setstate__ = interp2app(W_NDimArray.descr_setstate),
     __array_finalize__ = interp2app(W_NDimArray.descr___array_finalize__),
-
+    __array_prepare__ = interp2app(W_NDimArray.descr___array_prepare__),
+    __array_wrap__ = interp2app(W_NDimArray.descr___array_wrap__),
     __array__         = interp2app(W_NDimArray.descr___array__),
 )
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -342,6 +342,9 @@
         if w_ldtype.is_str_type() and w_rdtype.is_str_type() and \
            self.comparison_func:
             pass
+        elif (w_ldtype.is_str_type() or w_rdtype.is_str_type()) and \
+            self.comparison_func and w_out is None:
+            return space.wrap(False)
         elif (w_ldtype.is_flexible_type() or \
                 w_rdtype.is_flexible_type()):
             raise OperationError(space.w_TypeError, space.wrap(
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -712,7 +712,8 @@
 
     def test_comparisons(self):
         import operator
-        from numpypy import equal, not_equal, less, less_equal, greater, greater_equal
+        from numpypy import (equal, not_equal, less, less_equal, greater,
+                            greater_equal, arange)
 
         for ufunc, func in [
             (equal, operator.eq),
@@ -735,7 +736,9 @@
                 (3, 3.5),
             ]:
                 assert ufunc(a, b) == func(a, b)
-
+        c = arange(10)
+        val = c == 'abcdefg'
+        assert val == False
 
     def test_count_nonzero(self):
         from numpypy import count_nonzero
diff --git a/rpython/tool/gcanalyze.py b/rpython/tool/gcanalyze.py
new file mode 100755
--- /dev/null
+++ b/rpython/tool/gcanalyze.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+""" Parse gcdumps. Use by saying PYPYLOG=gc-collect:log pypy <program>
+and run it by:
+
+gcanalyze.py logfile [--plot]
+"""
+
+import sys
+from rpython.tool.logparser import parse_log
+
+NO_BUCKETS = 8
+
+def main(arg):
+    log = parse_log(open(arg).readlines())
+    all = []
+    for entry in log:
+        if entry[0].startswith('gc-collect'):
+            start = entry[1]
+            end = entry[2]
+            all.append(float(end - start) / 1000000)
+    avg = sum(all) / len(all)
+    max_t = max(all)
+    print "AVG:", "%.1fms" % avg, "MAX:", "%.1fms" % max_t
+    buckets = [0] * (NO_BUCKETS + 1)
+    for item in all:
+        bucket = int(item / max_t * NO_BUCKETS)
+        buckets[bucket] += 1
+    l1 = ["%.1fms" % ((i + 1) * max_t / NO_BUCKETS) for i in range(NO_BUCKETS)]
+    l2 = [str(i) for i in buckets[1:]]
+    for i, elem in enumerate(l1):
+        l2[i] += " " * (len(elem) - len(l2[i]))
+    print " ".join(l1)
+    print " ".join(l2)
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2 or len(sys.argv) > 3:
+        print __doc__
+        sys.exit(1)
+    plot = False
+    if len(sys.argv) == 3:
+        if sys.argv[1] == '--plot':
+            plot = True
+            arg = sys.argv[2]
+        elif sys.argv[2] == '--plot':
+            plot = True
+            arg = sys.argv[1]
+        else:
+            print "Wrong command line options:", sys.argv
+            sys.exit(1)
+    else:
+        arg = sys.argv[1]
+    main(arg)

From noreply at buildbot.pypy.org  Mon Oct 14 14:53:40 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 14:53:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: refactor this script a little
Message-ID: <20131014125341.004981D22CC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67356:9bee4fc9c02b
Date: 2013-10-14 14:26 +0200
http://bitbucket.org/pypy/pypy/changeset/9bee4fc9c02b/

Log:	refactor this script a little

diff --git a/rpython/tool/gcanalyze.py b/rpython/tool/gcanalyze.py
--- a/rpython/tool/gcanalyze.py
+++ b/rpython/tool/gcanalyze.py
@@ -18,15 +18,20 @@
             start = entry[1]
             end = entry[2]
             all.append(float(end - start) / 1000000)
+    format_output(all)
+
+def format_output(all):
     avg = sum(all) / len(all)
     max_t = max(all)
-    print "AVG:", "%.1fms" % avg, "MAX:", "%.1fms" % max_t
-    buckets = [0] * (NO_BUCKETS + 1)
+    print "AVG:", "%.1fms" % avg, "MAX:", "%.1fms" % max_t, "TOTAL:" , "%.1fms" % sum(all)
+    buckets = [0] * NO_BUCKETS
     for item in all:
         bucket = int(item / max_t * NO_BUCKETS)
+        if bucket == len(buckets):
+            bucket = len(buckets) - 1
         buckets[bucket] += 1
     l1 = ["%.1fms" % ((i + 1) * max_t / NO_BUCKETS) for i in range(NO_BUCKETS)]
-    l2 = [str(i) for i in buckets[1:]]
+    l2 = [str(i) for i in buckets]
     for i, elem in enumerate(l1):
         l2[i] += " " * (len(elem) - len(l2[i]))
     print " ".join(l1)

From noreply at buildbot.pypy.org  Mon Oct 14 14:53:42 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 14:53:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131014125342.29C7E1D22D0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67357:5c7098ff9525
Date: 2013-10-14 14:39 +0200
http://bitbucket.org/pypy/pypy/changeset/5c7098ff9525/

Log:	merge

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -129,6 +129,7 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
+            self.pure(rop.INT_ADD, [op.getarg(1), op.getarg(0)], op.result)
             # Synthesize the reverse op for optimize_default to reuse
             self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
             self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -3670,6 +3670,20 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_int_add_commutative(self):
+        ops = """
+        [i0, i1]
+        i2 = int_add(i0, i1)
+        i3 = int_add(i1, i0)
+        jump(i2, i3)
+        """
+        expected = """
+        [i0, i1]
+        i2 = int_add(i0, i1)
+        jump(i2, i2)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_framestackdepth_overhead(self):
         ops = """
         [p0, i22]

From noreply at buildbot.pypy.org  Mon Oct 14 16:03:49 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 14 Oct 2013 16:03:49 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: Add stm_transaction_break after CALL
 ops in stmrewrite. That way
Message-ID: <20131014140349.957C11C067F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67358:e22195363636
Date: 2013-10-14 16:03 +0200
http://bitbucket.org/pypy/pypy/changeset/e22195363636/

Log:	Add stm_transaction_break after CALL ops in stmrewrite. That way
	inevitable transactions causes by the calls will be much shorter and
	less likely to go on into atomic blocks.

diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -45,6 +45,7 @@
     def rewrite(self, operations):
         # overridden method from parent class
         #
+        insert_transaction_break = False
         for op in operations:
             if not we_are_translated():
                 # only possible in tests:
@@ -81,6 +82,19 @@
             # ----------  pure operations, guards  ----------
             if op.is_always_pure() or op.is_guard() or op.is_ovf():
                 self.newops.append(op)
+
+                # insert a transaction break after call_release_gil
+                # in order to commit the inevitable transaction following
+                # it immediately
+                if (op.getopnum() == rop.GUARD_NOT_FORCED
+                    and insert_transaction_break):
+                    # insert transaction_break after GUARD after call
+                    self.newops.append(
+                        ResOperation(rop.STM_TRANSACTION_BREAK, [], None))
+                    insert_transaction_break = False
+                else:
+                    assert insert_transaction_break is False
+
                 continue
             # ----------  getfields  ----------
             if op.getopnum() in (rop.GETFIELD_GC,
@@ -104,9 +118,17 @@
                 continue
             # ----------  calls  ----------
             if op.is_call():
+                if (op.getopnum() == rop.CALL_MAY_FORCE or
+                    op.getopnum() == rop.CALL_ASSEMBLER or
+                    op.getopnum() == rop.CALL_RELEASE_GIL):
+                    # insert more transaction breaks after function
+                    # calls since they are likely to return as
+                    # inevitable transactions
+                    insert_transaction_break = True
+                    
                 if op.getopnum() == rop.CALL_RELEASE_GIL:
                     # self.fallback_inevitable(op)
-                    # done by assembler._release_gil_shadowstack()
+                    # is done by assembler._release_gil_shadowstack()
                     self.newops.append(op)
                 elif op.getopnum() == rop.CALL_ASSEMBLER:
                     self.handle_call_assembler(op)
@@ -158,6 +180,9 @@
             # ----------  fall-back  ----------
             self.fallback_inevitable(op)
             #
+
+        # call_XX without guard_not_forced?
+        assert not insert_transaction_break
         return self.newops
 
     def write_to_read_categories(self):
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -82,6 +82,7 @@
                 []
                 %s
                 call(123, descr=cd)
+                stm_transaction_break()
                 jump()
             """ % ("$INEV" if inev else "",), cd=calldescr)
     
@@ -94,6 +95,7 @@
             [p1, p2]
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, p2, descr=tzdescr)
+            stm_transaction_break()
             jump()
         """)
 
@@ -109,6 +111,7 @@
             p3 = same_as(ConstPtr(t))
             cond_call_stm_b(p3, descr=P2Wdescr)
             setfield_gc(p3, p2, descr=tzdescr)
+            stm_transaction_break()
             jump()
             """, t=NULL)
 
@@ -133,6 +136,7 @@
             setfield_gc(p2, p0, descr=tzdescr)
             cond_call_stm_b(p1, descr=P2Rdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
+            stm_transaction_break()
             jump()
             """, t=NULL)
 
@@ -155,6 +159,7 @@
             setfield_gc(p2, p0, descr=tzdescr)
             cond_call_stm_b(p1, descr=P2Rdescr)
             p5 = getfield_gc(p1, descr=tzdescr)
+            stm_transaction_break()
             jump()
         """)
 
@@ -178,6 +183,7 @@
             setfield_gc(p2, p0, descr=tzdescr)
             cond_call_stm_b(p1, descr=P2Rdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
+            stm_transaction_break()
             jump()
         """)
 
@@ -203,6 +209,7 @@
             setfield_gc(p2, p0, descr=tzdescr)
             cond_call_stm_b(p1, descr=P2Rdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
+            stm_transaction_break()
             jump()
         """
         for op in ops:
@@ -225,6 +232,7 @@
                                 descr=malloc_big_fixedsize_descr)
             cond_call_stm_b(p3, descr=P2Wdescr)
             setfield_gc(p3, p1, descr=tzdescr)
+            stm_transaction_break()
             jump(p2)
         """)
 
@@ -243,6 +251,7 @@
                                 %(tdescr.size)d, %(tdescr.tid)d, \
                                 descr=malloc_big_fixedsize_descr)
             p4 = getfield_gc(p1, descr=tzdescr)
+            stm_transaction_break()
             jump(p2)
         """)
             
@@ -258,6 +267,7 @@
                                 %(tdescr.size)d, %(tdescr.tid)d, \
                                 descr=malloc_big_fixedsize_descr)
             setfield_gc(p2, p1, descr=tzdescr)
+            stm_transaction_break()
             jump(p2)
         """)
 
@@ -273,6 +283,7 @@
             setfield_gc(p1, p2, descr=tzdescr)
             cond_call_stm_b(p3, descr=P2Wdescr)
             setfield_gc(p3, p4, descr=tzdescr)
+            stm_transaction_break()
             jump()
         """)
 
@@ -287,6 +298,7 @@
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, p2, descr=tzdescr)
             setfield_gc(p1, i3, descr=tydescr)
+            stm_transaction_break()
             jump()
         """)
 
@@ -304,6 +316,7 @@
             label(p1, i3)
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, i3, descr=tydescr)
+            stm_transaction_break()
             jump(p1)
         """)
 
@@ -314,6 +327,7 @@
             jump()
         """, """
             [i1, i2]
+            stm_transaction_break()
             jump()
         """)
 
@@ -333,7 +347,7 @@
             testcase = """
                 [i1, i2, p1, p2, f1]
                 %s
-                jump(i2)
+                finish()
             """ % op
             self.check_rewrite(testcase, testcase)
 
@@ -346,6 +360,7 @@
             [p1]
             cond_call_stm_b(p1, descr=P2Rdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
+            stm_transaction_break()
             jump(p2)
         """)
 
@@ -361,6 +376,7 @@
             p3 = same_as(ConstPtr(t))
             cond_call_stm_b(p3, descr=P2Rdescr)
             p2 = getfield_gc(p3, descr=tzdescr)
+            stm_transaction_break()
             jump(p2)
         """, t=NULL)
         # XXX could do better: G2Rdescr
@@ -374,6 +390,7 @@
             [p1, i2]
             cond_call_stm_b(p1, descr=P2Rdescr)
             i3 = getarrayitem_gc(p1, i2, descr=adescr)
+            stm_transaction_break()
             jump(i3)
         """)
 
@@ -386,6 +403,7 @@
             [p1, i2]
             cond_call_stm_b(p1, descr=P2Rdescr)
             i3 = getinteriorfield_gc(p1, i2, descr=adescr)
+            stm_transaction_break()
             jump(i3)
         """)
 
@@ -400,6 +418,7 @@
             cond_call_stm_b(p1, descr=P2Rdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
             i2 = getfield_gc(p1, descr=tydescr)
+            stm_transaction_break()
             jump(p2, i2)
         """)
 
@@ -415,6 +434,7 @@
             p2 = getfield_gc(p1, descr=tzdescr)
             cond_call_stm_b(p2, descr=P2Rdescr)
             i2 = getfield_gc(p2, descr=tydescr)
+            stm_transaction_break()
             jump(p2, i2)
         """)
 
@@ -434,6 +454,7 @@
             i2 = int_add(i1, 1)
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, i2, descr=tydescr)
+            stm_transaction_break()
             jump(p1)
         """)
 
@@ -448,6 +469,7 @@
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, 123, descr=tydescr)
             p2 = getfield_gc(p1, descr=tzdescr)
+            stm_transaction_break()
             jump(p2)
         """)
 
@@ -463,6 +485,7 @@
                                 %(tdescr.size)d, %(tdescr.tid)d, \
                                 descr=malloc_big_fixedsize_descr)
             p2 = getfield_gc(p1, descr=tzdescr)
+            stm_transaction_break()
             jump(p2)
         """)
 
@@ -488,6 +511,7 @@
             call(p2, descr=calldescr1)
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, 5, descr=tydescr)
+            stm_transaction_break()
             jump(p2)
         """, calldescr1=calldescr1)
 
@@ -504,6 +528,7 @@
             i3 = getfield_raw(i1, descr=tydescr)
             keepalive(i3)
             i4 = getfield_raw(i2, descr=tydescr)
+            stm_transaction_break()
             jump(i3, i4)
         """)
 
@@ -519,6 +544,7 @@
         """, """
             [i1]
             i2 = getfield_raw(i1, descr=fdescr)
+            stm_transaction_break()
             jump(i2)
         """, fdescr=fdescr)
 
@@ -536,6 +562,7 @@
             label(i1, i2, i3)
             $INEV
             i4 = getfield_raw(i2, descr=tydescr)
+            stm_transaction_break()
             jump(i3, i4)
         """)
 
@@ -550,6 +577,7 @@
             $INEV
             i3 = getarrayitem_raw(i1, 5, descr=adescr)
             i4 = getarrayitem_raw(i2, i3, descr=adescr)
+            stm_transaction_break()
             jump(i3, i4)
         """)
 
@@ -565,6 +593,7 @@
             setarrayitem_gc(p1, i1, p2, descr=adescr)
             cond_call_stm_b(p3, descr=P2Wdescr)
             setarrayitem_gc(p3, i3, p4, descr=adescr)
+            stm_transaction_break()
             jump()
         """)
 
@@ -581,6 +610,7 @@
             setarrayitem_gc(p1, i2, p2, descr=adescr)
             i4 = read_timestamp()
             setarrayitem_gc(p1, i3, p3, descr=adescr)
+            stm_transaction_break()
             jump()
         """)
 
@@ -597,6 +627,7 @@
             setinteriorfield_gc(p1, i2, p2, descr=adescr)
             i4 = read_timestamp()
             setinteriorfield_gc(p1, i3, p3, descr=adescr)
+            stm_transaction_break()
             jump()
         """)
 
@@ -611,11 +642,35 @@
             cond_call_stm_b(p1, descr=P2Wdescr)
             strsetitem(p1, i2, i3)
             unicodesetitem(p1, i2, i3)
+            stm_transaction_break()
             jump()
         """)
         py.test.skip("XXX not really right: should instead be an assert "
                      "that p1 is already a W")
 
+    def test_call_release_gil(self):
+        T = rffi.CArrayPtr(rffi.TIME_T)
+        calldescr2 = get_call_descr(self.gc_ll_descr, [T], rffi.TIME_T)
+        self.check_rewrite("""
+            [i1, i2, i3, p7]
+            setfield_gc(p7, 10, descr=tydescr)
+            call_release_gil(123, descr=calldescr2)
+            guard_not_forced() []
+            setfield_gc(p7, 20, descr=tydescr)
+            jump(i2, p7)
+        """, """
+            [i1, i2, i3, p7]
+            cond_call_stm_b(p7, descr=P2Wdescr)
+            setfield_gc(p7, 10, descr=tydescr)
+            call_release_gil(123, descr=calldescr2)
+            guard_not_forced() []
+            stm_transaction_break()
+            cond_call_stm_b(p7, descr=P2Wdescr)
+            setfield_gc(p7, 20, descr=tydescr)
+            stm_transaction_break()
+            jump(i2, p7)
+        """, calldescr2=calldescr2)
+        
     def test_fallback_to_inevitable(self):
         T = rffi.CArrayPtr(rffi.TIME_T)
         calldescr2 = get_call_descr(self.gc_ll_descr, [T], rffi.TIME_T)
@@ -623,7 +678,6 @@
             "setfield_raw(i1, i2, descr=tydescr)",
             "setarrayitem_raw(i1, i2, i3, descr=tydescr)",
             "setinteriorfield_raw(i1, i2, i3, descr=adescr)",
-            "call_release_gil(123, descr=calldescr2)",
             "escape(i1)",    # a generic unknown operation
             ]
         for op in oplist:
@@ -641,6 +695,7 @@
                 %s
                 cond_call_stm_b(p7, descr=P2Wdescr)
                 setfield_gc(p7, 20, descr=tydescr)
+                stm_transaction_break()
                 jump(i2, p7)
             """ % op, calldescr2=calldescr2)
 
@@ -654,6 +709,7 @@
             cond_call_stm_b(p2, descr=P2Wdescr)
             cond_call_stm_b(p1, descr=P2Rdescr)
             copystrcontent(p1, p2, i1, i2, i3)
+            stm_transaction_break()
             jump()
         """)
 
@@ -675,6 +731,7 @@
                 setfield_gc(p1, 10, descr=tydescr)
                 %s
                 setfield_gc(p1, 20, descr=tydescr)
+                stm_transaction_break()
                 jump(p1)
             """ % op)
 
@@ -685,26 +742,33 @@
         T = rffi.CArrayPtr(rffi.TIME_T)
         calldescr2 = get_call_descr(self.gc_ll_descr, [T], rffi.TIME_T,
                                     fakeextrainfo())
-        for op in ["call(123, descr=calldescr2)",
-                   "call_assembler(123, descr=casmdescr)",
-                   "call_may_force(123, descr=calldescr2)",
-                   "call_loopinvariant(123, descr=calldescr2)",
-                   ]:
+        for op, guarded in [
+                ("call(123, descr=calldescr2)", False),
+                ("call_assembler(123, descr=casmdescr)", True),
+                ("call_may_force(123, descr=calldescr2)", True),
+                ("call_loopinvariant(123, descr=calldescr2)", False),
+                ]:
+            guard = "guard_not_forced() []" if guarded else ""
+            tr_break = "stm_transaction_break()" if guarded else ""
             self.check_rewrite("""
                 [p1]
                 setfield_gc(p1, 10, descr=tydescr)
                 %s
+                %s
                 setfield_gc(p1, 20, descr=tydescr)
                 jump(p1)
-            """ % op, """
+            """ % (op, guard), """
                 [p1]
                 cond_call_stm_b(p1, descr=P2Wdescr)
                 setfield_gc(p1, 10, descr=tydescr)
                 %s
+                %s
+                %s
                 cond_call_stm_b(p1, descr=P2Wdescr)
                 setfield_gc(p1, 20, descr=tydescr)
+                stm_transaction_break()
                 jump(p1)
-            """ % op, calldescr2=calldescr2)
+            """ % (op, guard, tr_break), calldescr2=calldescr2)
 
     def test_ptr_eq_null(self):
         self.check_rewrite("""
@@ -714,6 +778,7 @@
         """, """
             [p1, p2]
             i1 = ptr_eq(p1, NULL)
+            stm_transaction_break()
             jump(i1)
         """)
 
@@ -725,6 +790,7 @@
         """, """
             [p1, p2]
             i1 = ptr_eq(p1, p2)
+            stm_transaction_break()
             jump(i1)
         """)
 
@@ -736,6 +802,7 @@
         """, """
             [p1, p2]
             i1 = instance_ptr_eq(p1, p2)
+            stm_transaction_break()
             jump(i1)
         """)
 
@@ -747,6 +814,7 @@
         """, """
             [p1, p2]
             i1 = ptr_ne(p1, p2)
+            stm_transaction_break()
             jump(i1)
         """)
 
@@ -758,6 +826,7 @@
         """, """
             [p1, p2]
             i1 = instance_ptr_ne(p1, p2)
+            stm_transaction_break()
             jump(i1)
         """)
 

From noreply at buildbot.pypy.org  Mon Oct 14 16:04:36 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 16:04:36 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: make it signed
Message-ID: <20131014140436.5DD0F1D22CC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: incremental-gc
Changeset: r67359:5419d0bfff1f
Date: 2013-10-14 16:03 +0200
http://bitbucket.org/pypy/pypy/changeset/5419d0bfff1f/

Log:	make it signed

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1779,7 +1779,7 @@
             estimate_from_nursery = self.nursery_surviving_size * 2
             if estimate_from_nursery > estimate:
                 estimate = estimate_from_nursery
-            self.visit_all_objects_step(estimate)
+            self.visit_all_objects_step(intmask(estimate))
 
             # XXX A simplifying assumption that should be checked,
             # finalizers/weak references are rare and short which means that

From noreply at buildbot.pypy.org  Mon Oct 14 16:42:06 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 14 Oct 2013 16:42:06 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Fix the debugging logic.
Message-ID: <20131014144206.38E371C02C2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67360:cb84cf17db1c
Date: 2013-10-14 16:41 +0200
http://bitbucket.org/pypy/pypy/changeset/cb84cf17db1c/

Log:	Fix the debugging logic.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1083,6 +1083,8 @@
             pass    # black -> black
         elif self._debug_objects_to_trace_dict.contains(obj):
             pass    # black -> gray
+        elif self.header(obj).tid & GCFLAG_NO_HEAP_PTRS != 0:
+            pass    # black -> white-but-prebuilt-so-dont-care
         else:
             ll_assert(False, "black -> white pointer found")
 

From noreply at buildbot.pypy.org  Mon Oct 14 16:48:23 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 14 Oct 2013 16:48:23 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: work on stm blog post
Message-ID: <20131014144823.9FF7A1C02E2@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: extradoc
Changeset: r5070:2ffa39561b95
Date: 2013-10-14 16:48 +0200
http://bitbucket.org/pypy/extradoc/changeset/2ffa39561b95/

Log:	work on stm blog post

diff --git a/blog/draft/stm-sept2013.rst b/blog/draft/stm-oct2013.rst
rename from blog/draft/stm-sept2013.rst
rename to blog/draft/stm-oct2013.rst
--- a/blog/draft/stm-sept2013.rst
+++ b/blog/draft/stm-oct2013.rst
@@ -12,11 +12,11 @@
 
 But that is not all. Right after the sprint, we were able to squeeze
 the last obvious bugs in the STM-JIT combination. However, the performance
-was nowhere near what we want. So until now, we fixed some of the most
+was nowhere near to what we want. So until now, we fixed some of the most
 obvious issues. Many come from RPython erring on the side of caution
 and e.g. making a transaction inevitable even if that is not strictly
 necessary, thereby limiting parallelism.
-**XXX any interesting details?**
+**XXX any interesting details? transaction breaks maybe? guard counters?**
 There are still many performance issues of various complexity left
 to tackle. So stay tuned or contribute :)
 
@@ -24,23 +24,42 @@
 show you some numbers that are indicative of things to come.
 Our set of STM benchmarks is very small unfortunately 
 (something you can help us out with), so this is 
-not representative of real-world performance.
+not representative of real-world performance. We tried to
+minimize the effect of JIT warm-up in the benchmark results.
+
 
 **Raytracer** from `stm-benchmarks <https://bitbucket.org/Raemi/stm-benchmarks/src>`_:
-Render times for a 1024x1024 image using 6 threads
+Render times in seconds for a 1024x1024 image using 8 threads:
 
 +-------------+----------------------+
 | Interpeter  | Time (no-JIT / JIT)  |
 +=============+======================+
-| PyPy-2.1    | ... / ...            |
+| PyPy-2.1    | 148 / 2.56           |
 +-------------+----------------------+
-| CPython     | ... / -              |
+| CPython     | 73.4 / -             |
 +-------------+----------------------+
-| PyPy-STM    | ... / ...            |
+| PyPy-STM    | 87.0 / 10.8          |
 +-------------+----------------------+
 
-**XXX same for Richards**
+For comparison, the 3 interpreters in their best settings running
+single-threaded: 2.47, 81.1, 50.2
 
+**Richards** from `PyPy repository on the stmgc-c4
+branch <https://bitbucket.org/pypy/pypy/commits/branch/stmgc-c4>`_:
+Average time per iteration in milliseconds using 8 threads:
+
++-------------+----------------------+
+| Interpeter  | Time (no-JIT / JIT)  |
++=============+======================+
+| PyPy-2.1    | 492 / 15.4           |
++-------------+----------------------+
+| CPython     | 237 / -              |
++-------------+----------------------+
+| PyPy-STM    | 538 / 116            |
++-------------+----------------------+
+
+For comparison, the 3 interpreters in their best settings running
+single-threaded: 15.6, 239, 371
 
 All this can be found in the `PyPy repository on the stmgc-c4
 branch <https://bitbucket.org/pypy/pypy/commits/branch/stmgc-c4>`_.
@@ -50,3 +69,5 @@
 You can also download a prebuilt binary frome here: **XXX**
 
 
+
+

From noreply at buildbot.pypy.org  Mon Oct 14 16:56:39 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 16:56:39 +0200 (CEST)
Subject: [pypy-commit] buildbot default: add tosh to osx buildbots
Message-ID: <20131014145639.95FF21C02E2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r866:7caedcda221a
Date: 2013-10-14 16:56 +0200
http://bitbucket.org/pypy/buildbot/changeset/7caedcda221a/

Log:	add tosh to osx buildbots

diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -167,6 +167,7 @@
 JITLINUX32 = "pypy-c-jit-linux-x86-32"
 JITLINUX64 = "pypy-c-jit-linux-x86-64"
 JITMACOSX64 = "pypy-c-jit-macosx-x86-64"
+JITMACOSX64_2 = "pypy-c-jit-macosx-x86-64-2"
 JITWIN32 = "pypy-c-jit-win-x86-32"
 JITWIN64 = "pypy-c-jit-win-x86-64"
 JITFREEBSD764 = 'pypy-c-jit-freebsd-7-x86-64'
@@ -252,6 +253,7 @@
                         JITLINUX32,
                         JITLINUX64,
                         JITMACOSX64,
+                        JITMACOSX64_2,
                         JITWIN32,
                         JITWIN64,
                         JITFREEBSD764,
@@ -352,11 +354,17 @@
                    "category": 'mac32'
                   },
                   {"name" : JITMACOSX64,
-                   "slavenames": ["xerxes"],
+                   "slavenames": ["xerxes", "tosh"],
                    'builddir' : JITMACOSX64,
                    'factory' : pypyJITTranslatedTestFactoryOSX64,
                    'category' : 'mac64',
                    },
+                  {"name" : JITMACOSX64_2,
+                   "slavenames": ["xerxes", "tosh"],
+                   'builddir' : JITMACOSX64_2,
+                   'factory' : pypyJITTranslatedTestFactoryOSX64,
+                   'category' : 'mac64',
+                   },
                   {"name": WIN32,
                    "slavenames": ["aurora", "SalsaSalsa"],
                    "builddir": WIN32,

From noreply at buildbot.pypy.org  Mon Oct 14 17:01:48 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 14 Oct 2013 17:01:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: a test that tries to inline across
 multiple jit drivers. I don't understand why
Message-ID: <20131014150148.56D691C067F@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r67361:06e4649dd7a4
Date: 2013-10-14 17:00 +0200
http://bitbucket.org/pypy/pypy/changeset/06e4649dd7a4/

Log:	a test that tries to inline across multiple jit drivers. I don't
	understand why it fails

diff --git a/rpython/jit/metainterp/test/test_jitdriver.py b/rpython/jit/metainterp/test/test_jitdriver.py
--- a/rpython/jit/metainterp/test/test_jitdriver.py
+++ b/rpython/jit/metainterp/test/test_jitdriver.py
@@ -1,4 +1,5 @@
 """Tests for multiple JitDrivers."""
+import py
 from rpython.rlib.jit import JitDriver, unroll_safe, set_param
 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.jit.metainterp.warmspot import get_stats
@@ -142,6 +143,51 @@
         stats = get_stats()
         assert stats.aborted_keys == [None, None]
 
+    def test_inline_across_languages(self):
+        py.test.skip("why does this not work")
+        driver_weird = JitDriver(
+            greens = ["pc", "bc"],
+            reds = ["acc", "x", "y", "z"])
+
+        def interp1(bc, x, y, z):
+            pc = 0
+            acc = 0
+            while True:
+                driver_weird.jit_merge_point(bc=bc, pc=pc, acc=acc, x=x, y=y, z=z)
+                op = ord(bc[pc])
+                pc += 1
+                if op == 0:
+                    acc += x
+                if op == 1:
+                    acc += y
+                if op == 2:
+                    acc *= z
+                if op == 3:
+                    pc = 0
+                if pc >= len(bc):
+                    break
+            return acc
+
+        driver = JitDriver(
+                greens = ["substract"],
+                reds = ["x"],
+        )
+        def interp2(x):
+            substract = interp1('\x00', 0, 0, 0)
+            while True:
+                driver.jit_merge_point(substract=substract, x=x)
+                substract += 1
+                if x < 0:
+                    break
+                if substract == 10:
+                    # computes x + 1 * (-1)
+                    x = interp1('\x01\x02\x00', x, 1, -1)
+                    substract = 0
+        interp2(100)
+        self.meta_interp(interp2, [100], listcomp=True, backendopt=True,
+                         listops=True, inline=True)
+        self.check_resops(call_assembler=0)
+
 
 class TestLLtype(MultipleJitDriversTests, LLJitMixin):
     pass

From noreply at buildbot.pypy.org  Mon Oct 14 17:17:07 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 14 Oct 2013 17:17:07 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: my take
Message-ID: <20131014151707.BE36D1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5071:39251676181b
Date: 2013-10-14 17:16 +0200
http://bitbucket.org/pypy/extradoc/changeset/39251676181b/

Log:	my take

diff --git a/blog/draft/stm-oct2013.rst b/blog/draft/stm-oct2013.rst
--- a/blog/draft/stm-oct2013.rst
+++ b/blog/draft/stm-oct2013.rst
@@ -29,45 +29,50 @@
 
 
 **Raytracer** from `stm-benchmarks <https://bitbucket.org/Raemi/stm-benchmarks/src>`_:
-Render times in seconds for a 1024x1024 image using 8 threads:
+Render times in seconds for a 1024x1024 image:
 
-+-------------+----------------------+
-| Interpeter  | Time (no-JIT / JIT)  |
-+=============+======================+
-| PyPy-2.1    | 148 / 2.56           |
-+-------------+----------------------+
-| CPython     | 73.4 / -             |
-+-------------+----------------------+
-| PyPy-STM    | 87.0 / 10.8          |
-+-------------+----------------------+
++-------------+----------------------+-------------------+
+| Interpreter | Base time: 1 thread  | 8 threads         |
++=============+======================+===================+
+| PyPy-2.1    |    2.47              |     2.56          |
++-------------+----------------------+-------------------+
+| CPython     |    81.1              |     73.4          |
++-------------+----------------------+-------------------+
+| PyPy-STM    |    50.2              |     10.8          |
++-------------+----------------------+-------------------+
 
-For comparison, the 3 interpreters in their best settings running
-single-threaded: 2.47, 81.1, 50.2
+For comparison, disabling the JIT gives 148ms on PyPy-2.1 and 87ms on
+PyPy-STM (with 8 threads).
 
 **Richards** from `PyPy repository on the stmgc-c4
 branch <https://bitbucket.org/pypy/pypy/commits/branch/stmgc-c4>`_:
 Average time per iteration in milliseconds using 8 threads:
 
-+-------------+----------------------+
-| Interpeter  | Time (no-JIT / JIT)  |
-+=============+======================+
-| PyPy-2.1    | 492 / 15.4           |
-+-------------+----------------------+
-| CPython     | 237 / -              |
-+-------------+----------------------+
-| PyPy-STM    | 538 / 116            |
-+-------------+----------------------+
++-------------+----------------------+-------------------+
+| Interpreter | Base time: 1 thread  | 8 threads         |
++=============+======================+===================+
+| PyPy-2.1    |   15.6               |  15.4             |
++-------------+----------------------+-------------------+
+| CPython     |   239                |  237              |
++-------------+----------------------+-------------------+
+| PyPy-STM    |   371                |  116              |
++-------------+----------------------+-------------------+
 
-For comparison, the 3 interpreters in their best settings running
-single-threaded: 15.6, 239, 371
+For comparison, disabling the JIT gives 492ms on PyPy-2.1 and 538ms on
+PyPy-STM.
 
 All this can be found in the `PyPy repository on the stmgc-c4
 branch <https://bitbucket.org/pypy/pypy/commits/branch/stmgc-c4>`_.
 Try it for yourself, but keep in mind that this is still experimental
 with a lot of things yet to come.
 
-You can also download a prebuilt binary frome here: **XXX**
+You can also download a prebuilt binary from here: **XXX**
 
-
-
-
+As a summary, what the numbers tell us is that PyPy-STM is, as expected,
+the only of the three interpreters where multithreading gives a large
+improvement in speed.  What they also tell us is that, obviously, the
+result is not good enough *yet:* it still takes longer on a 8-threaded
+PyPy-STM than on a regular single-threaded PyPy-2.1.  As you should know
+by now, we are good at promizing speed and delivering it years later.
+It has been two years already since PyPy-STM started, so we're in the
+fast-progressing step right now :-)

From noreply at buildbot.pypy.org  Mon Oct 14 17:21:05 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 17:21:05 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: fix minor tests
Message-ID: <20131014152105.A57771C01B0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: incremental-gc
Changeset: r67362:b6950b0c9f5a
Date: 2013-10-14 17:20 +0200
http://bitbucket.org/pypy/pypy/changeset/b6950b0c9f5a/

Log:	fix minor tests

diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -618,11 +618,10 @@
         #at this point the first object should have been processed
         newobj = self.malloc(S)
         self.write(oldobj,'next',newobj)
-        #the barrier should have made the object gray
-        newhdr = self.gc.header(llmemory.cast_ptr_to_adr(newobj))
-        assert oldhdr.tid & incminimark.GCFLAG_GRAY
-        assert newhdr.tid & (incminimark.GCFLAG_VISITED | incminimark.GCFLAG_GRAY) == 0
-        #checks gray object is in objects_to_trace
+
+        assert self.gc.header(self.gc.old_objects_pointing_to_young.tolist()[0]) == oldhdr
+
+        self.gc.minor_collection()
         self.gc.debug_check_consistency()
 
     def test_sweeping_simple(self):
@@ -635,7 +634,7 @@
             curobj.x = i
             self.stackroots.append(curobj)
 
-        self.gc.debug_gc_step_until(incminimark.STATE_SWEEPING_RAWMALLOC)
+        self.gc.debug_gc_step_until(incminimark.STATE_SWEEPING)
         oldobj = self.stackroots[-1]
         oldhdr = self.gc.header(llmemory.cast_ptr_to_adr(oldobj))
         assert oldhdr.tid & incminimark.GCFLAG_VISITED
@@ -650,46 +649,6 @@
         newobj1 = oldobj.next
         assert newobj1.x == 1337
 
-    def test_young_gray_collected(self):
-        from rpython.memory.gc import incminimark
-
-        # Test the write barrier triggers on a young object
-        # but doesnt crash when that object is collected
-
-        for i in range(2):
-            curobj = self.malloc(S)
-            curobj.x = i
-            self.stackroots.append(curobj)
-
-
-        self.gc.debug_gc_step_until(incminimark.STATE_MARKING)
-
-        self.gc.minor_collection()
-        self.gc.visit_all_objects_step(1)
-
-        oldobj = self.stackroots[-1]
-
-        newobj = self.malloc(S)
-        newobj.x = 5
-        # make newobj gray
-        self.write(oldobj,'next',newobj)
-        #the barrier should have made the object gray
-        newhdr = self.gc.header(llmemory.cast_ptr_to_adr(newobj))
-        assert newhdr.tid & incminimark.GCFLAG_GRAY == 0
-        assert (self.gc.header(llmemory.cast_ptr_to_adr(oldobj)).tid &
-                incminimark.GCFLAG_GRAY)
-
-        assert self.gc.gc_state == incminimark.STATE_MARKING
-        # make newobj unreachable again
-        self.write(oldobj,'next',oldobj)
-
-        #complete collection
-        self.gc.debug_gc_step_until(incminimark.STATE_SCANNING)
-        self.gc.debug_check_consistency()
-
-        # now object is collected
-        assert py.test.raises(RuntimeError,"newobj.x")
-
     # Test trying to be a bit comprehensive about
     # states and types of objects
     def test_allocate_states(self):
@@ -722,7 +681,7 @@
         unreachable = []
 
         while True:
-
+            
             if self.gc.gc_state not in nallocated:
                 nallocated[self.gc.gc_state] = 0
 

From noreply at buildbot.pypy.org  Mon Oct 14 17:50:06 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 14 Oct 2013 17:50:06 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: Fix for
 test_incminimark_gc.py:test_writebarrier_before_copy
Message-ID: <20131014155006.B7FDF1C01F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: incremental-gc
Changeset: r67363:a20b1926aad6
Date: 2013-10-14 17:49 +0200
http://bitbucket.org/pypy/pypy/changeset/a20b1926aad6/

Log:	Fix for test_incminimark_gc.py:test_writebarrier_before_copy

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1513,6 +1513,13 @@
                         interval_start = interval_stop
                         cardbyte >>= 1
                     interval_start = next_byte_start
+                #
+                # If we're incrementally marking right now, sorry, we also
+                # need to add the object to 'objects_to_trace' and have it
+                # fully traced very soon.
+                if self.gc_state == STATE_MARKING:
+                    self.header(obj).tid &= ~GCFLAG_VISITED
+                    self.objects_to_trace.append(obj)
 
 
     def collect_oldrefs_to_nursery(self):

From noreply at buildbot.pypy.org  Mon Oct 14 17:57:33 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 17:57:33 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: tweaks
Message-ID: <20131014155733.5580E1C02C2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5072:2158303e96f9
Date: 2013-10-04 11:25 +0200
http://bitbucket.org/pypy/extradoc/changeset/2158303e96f9/

Log:	tweaks

diff --git a/talk/pyconza2013/php/talk.pdf b/talk/pyconza2013/php/talk.pdf
index c3455b8c36fbba1f91ef50062af1704ccdce201f..98dc3124127d47e75c3265399dc8de90b47ab02e
GIT binary patch

[cut]

diff --git a/talk/pyconza2013/php/talk.rst b/talk/pyconza2013/php/talk.rst
--- a/talk/pyconza2013/php/talk.rst
+++ b/talk/pyconza2013/php/talk.rst
@@ -11,16 +11,18 @@
 
    \maketitle
 
-introduction
+Introduction
 ------------
 
-* me - Maciej Fija&#322;kowski, PyPy core developer
+* Maciej Fija&#322;kowski, PyPy core developer
 
-* technology - PyPy
+* PHP interpreter
 
-* project - PHP interpreter
+* PyPy technology
 
-Wait, what???!!!1
+* if you have a question, feel free to interrupt me
+
+PHP - the problem
 -----------------
 
 * PHP - by far the most popular language on the web

From noreply at buildbot.pypy.org  Mon Oct 14 17:57:34 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 17:57:34 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: merge
Message-ID: <20131014155734.9F83E1C02C2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5073:8ae7445f165e
Date: 2013-10-14 17:57 +0200
http://bitbucket.org/pypy/extradoc/changeset/8ae7445f165e/

Log:	merge

diff --git a/blog/draft/stm-oct2013.rst b/blog/draft/stm-oct2013.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/stm-oct2013.rst
@@ -0,0 +1,78 @@
+Update on STM
+=============
+
+Hi all,
+
+the sprint in London was a lot of fun and very fruitful. In the last
+update on STM, Armin was working on improving and specializing the 
+automatic barrier placement.
+There is still a lot to do in that area, but that work was merged and
+lowered the overhead of STM over non-STM to around **XXX**. The same
+improvement has still to be done in the JIT.
+
+But that is not all. Right after the sprint, we were able to squeeze
+the last obvious bugs in the STM-JIT combination. However, the performance
+was nowhere near to what we want. So until now, we fixed some of the most
+obvious issues. Many come from RPython erring on the side of caution
+and e.g. making a transaction inevitable even if that is not strictly
+necessary, thereby limiting parallelism.
+**XXX any interesting details? transaction breaks maybe? guard counters?**
+There are still many performance issues of various complexity left
+to tackle. So stay tuned or contribute :)
+
+Now, since the JIT is all about performance, we want to at least 
+show you some numbers that are indicative of things to come.
+Our set of STM benchmarks is very small unfortunately 
+(something you can help us out with), so this is 
+not representative of real-world performance. We tried to
+minimize the effect of JIT warm-up in the benchmark results.
+
+
+**Raytracer** from `stm-benchmarks <https://bitbucket.org/Raemi/stm-benchmarks/src>`_:
+Render times in seconds for a 1024x1024 image:
+
++-------------+----------------------+-------------------+
+| Interpreter | Base time: 1 thread  | 8 threads         |
++=============+======================+===================+
+| PyPy-2.1    |    2.47              |     2.56          |
++-------------+----------------------+-------------------+
+| CPython     |    81.1              |     73.4          |
++-------------+----------------------+-------------------+
+| PyPy-STM    |    50.2              |     10.8          |
++-------------+----------------------+-------------------+
+
+For comparison, disabling the JIT gives 148ms on PyPy-2.1 and 87ms on
+PyPy-STM (with 8 threads).
+
+**Richards** from `PyPy repository on the stmgc-c4
+branch <https://bitbucket.org/pypy/pypy/commits/branch/stmgc-c4>`_:
+Average time per iteration in milliseconds using 8 threads:
+
++-------------+----------------------+-------------------+
+| Interpreter | Base time: 1 thread  | 8 threads         |
++=============+======================+===================+
+| PyPy-2.1    |   15.6               |  15.4             |
++-------------+----------------------+-------------------+
+| CPython     |   239                |  237              |
++-------------+----------------------+-------------------+
+| PyPy-STM    |   371                |  116              |
++-------------+----------------------+-------------------+
+
+For comparison, disabling the JIT gives 492ms on PyPy-2.1 and 538ms on
+PyPy-STM.
+
+All this can be found in the `PyPy repository on the stmgc-c4
+branch <https://bitbucket.org/pypy/pypy/commits/branch/stmgc-c4>`_.
+Try it for yourself, but keep in mind that this is still experimental
+with a lot of things yet to come.
+
+You can also download a prebuilt binary from here: **XXX**
+
+As a summary, what the numbers tell us is that PyPy-STM is, as expected,
+the only of the three interpreters where multithreading gives a large
+improvement in speed.  What they also tell us is that, obviously, the
+result is not good enough *yet:* it still takes longer on a 8-threaded
+PyPy-STM than on a regular single-threaded PyPy-2.1.  As you should know
+by now, we are good at promizing speed and delivering it years later.
+It has been two years already since PyPy-STM started, so we're in the
+fast-progressing step right now :-)
diff --git a/blog/draft/stm-sept2013.rst b/blog/draft/stm-sept2013.rst
deleted file mode 100644
--- a/blog/draft/stm-sept2013.rst
+++ /dev/null
@@ -1,52 +0,0 @@
-Update on STM
-=============
-
-Hi all,
-
-the sprint in London was a lot of fun and very fruitful. In the last
-update on STM, Armin was working on improving and specializing the 
-automatic barrier placement.
-There is still a lot to do in that area, but that work was merged and
-lowered the overhead of STM over non-STM to around **XXX**. The same
-improvement has still to be done in the JIT.
-
-But that is not all. Right after the sprint, we were able to squeeze
-the last obvious bugs in the STM-JIT combination. However, the performance
-was nowhere near what we want. So until now, we fixed some of the most
-obvious issues. Many come from RPython erring on the side of caution
-and e.g. making a transaction inevitable even if that is not strictly
-necessary, thereby limiting parallelism.
-**XXX any interesting details?**
-There are still many performance issues of various complexity left
-to tackle. So stay tuned or contribute :)
-
-Now, since the JIT is all about performance, we want to at least 
-show you some numbers that are indicative of things to come.
-Our set of STM benchmarks is very small unfortunately 
-(something you can help us out with), so this is 
-not representative of real-world performance.
-
-**Raytracer** from `stm-benchmarks <https://bitbucket.org/Raemi/stm-benchmarks/src>`_:
-Render times for a 1024x1024 image using 6 threads
-
-+-------------+----------------------+
-| Interpeter  | Time (no-JIT / JIT)  |
-+=============+======================+
-| PyPy-2.1    | ... / ...            |
-+-------------+----------------------+
-| CPython     | ... / -              |
-+-------------+----------------------+
-| PyPy-STM    | ... / ...            |
-+-------------+----------------------+
-
-**XXX same for Richards**
-
-
-All this can be found in the `PyPy repository on the stmgc-c4
-branch <https://bitbucket.org/pypy/pypy/commits/branch/stmgc-c4>`_.
-Try it for yourself, but keep in mind that this is still experimental
-with a lot of things yet to come.
-
-You can also download a prebuilt binary frome here: **XXX**
-
-
diff --git a/planning/jit.txt b/planning/jit.txt
--- a/planning/jit.txt
+++ b/planning/jit.txt
@@ -45,9 +45,6 @@
   (SETINTERIORFIELD, GETINTERIORFIELD). This is needed for the previous item to
   fully work.
 
-- {}.update({}) is not fully unrolled and constant folded because HeapCache
-  loses track of values in virtual-to-virtual ARRAY_COPY calls.
-
 - ovfcheck(a << b) will do ``result >> b`` and check that the result is equal
   to ``a``, instead of looking at the x86 flags.
 
diff --git a/talk/pyconza2013/Makefile b/talk/pyconza2013/Makefile
--- a/talk/pyconza2013/Makefile
+++ b/talk/pyconza2013/Makefile
@@ -1,13 +1,13 @@
 
 
 view: talk.pdf
-	xpdf talk.pdf
+	evince talk.pdf
 
 talk.pdf: talk.tex
 	64bit pdflatex talk.tex
 
-talk.tex: talk1.tex fix.py
-	python fix.py < talk1.tex > talk.tex
+talk.tex: talk.rst
+	rst2beamer --stylesheet=stylesheet.latex --documentoptions=14pt --input-encoding=utf8 --output-encoding=utf8 --overlaybullets=false $< > talk.tex
 
-talk1.tex: talk.rst
-	rst2beamer $< > talk1.tex
+clean:
+	rm -f talk.tex talk.pdf
diff --git a/talk/pyconza2013/stylesheet.latex b/talk/pyconza2013/stylesheet.latex
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2013/stylesheet.latex
@@ -0,0 +1,10 @@
+\usetheme{Warsaw}
+\usecolortheme{whale}
+\setbeamercovered{transparent}
+\definecolor{darkgreen}{rgb}{0, 0.5, 0.0}
+\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor}
+\addtobeamertemplate{block begin}{}{\setlength{\parskip}{35pt plus 1pt minus 1pt}}
+
+\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\red}[1]{\color{red}#1\normalcolor}
diff --git a/talk/pyconza2013/talk.pdf b/talk/pyconza2013/talk.pdf
index 6fed83a5c845e1d71cd4c32a98eb6a6b93d07bcf..fec69aacfbd0fc9af5c9c60eb65501eed188fc5a
GIT binary patch

[cut]

diff --git a/talk/pyconza2013/talk.rst b/talk/pyconza2013/talk.rst
--- a/talk/pyconza2013/talk.rst
+++ b/talk/pyconza2013/talk.rst
@@ -1,25 +1,25 @@
 .. include:: beamerdefs.txt
 
-=======================================
-Software Transactional Memory with PyPy
-=======================================
+.. raw:: latex
 
+   \title{Software Transactional Memory with PyPy}
+   \author[arigo]{Armin Rigo}
 
-Software Transactional Memory with PyPy
----------------------------------------
+   \institute{PyCon ZA 2013}
+   \date{4th October 2013}
 
-* PyCon ZA 2013
-
-* talk by Armin Rigo
-
-* sponsored by crowdfunding (thanks!)
+   \maketitle
 
 
 Introduction
 ------------
 
+* me: Armin Rigo
+
 * what is PyPy: an alternative implementation of Python
 
+* very compatible
+
 * main focus is on speed
 
 
@@ -27,13 +27,21 @@
 ------------
 
 .. image:: speed.png
-   :scale: 65%
+   :scale: 67%
    :align: center
 
 
 SQL by example
 --------------
 
+.. raw:: latex
+
+   %empty
+
+
+SQL by example
+--------------
+
 ::
 
     BEGIN TRANSACTION;
@@ -58,6 +66,27 @@
 
 ::
 
+    ...
+    obj.value += 1
+    ...
+
+
+Python by example
+-----------------
+
+::
+
+    ...
+    x = obj.value
+    obj.value = x + 1
+    ...
+
+
+Python by example
+-----------------
+
+::
+
     begin_transaction()
     x = obj.value
     obj.value = x + 1
@@ -100,10 +129,10 @@
 
 ::
 
-    BEGIN TRANSACTION;    BEGIN TRANSACTION;    BEGIN..
-    SELECT * FROM ...;    SELECT * FROM ...;    SELEC..
-    UPDATE ...;           UPDATE ...;           UPDAT..
-    COMMIT;               COMMIT;               COMMI..
+    BEGIN TRANSACTION;  BEGIN TRANSACTION;    BEGIN..
+    SELECT * FROM ...;  SELECT * FROM ...;    SELEC..
+    UPDATE ...;         UPDATE ...;           UPDAT..
+    COMMIT;             COMMIT;               COMMI..
 
 
 Locks != Transactions
@@ -111,9 +140,9 @@
 
 ::
 
-    with the_lock:        with the_lock:        with ..
-      x = obj.val           x = obj.val           x =..
-      obj.val = x + 1       obj.val = x + 1       obj..
+    with the_lock:     with the_lock:        with ..
+      x = obj.val        x = obj.val           x =..
+      obj.val = x + 1    obj.val = x + 1       obj..
 
 
 Locks != Transactions
@@ -121,9 +150,9 @@
 
 ::
 
-    with atomic:          with atomic:          with ..
-      x = obj.val           x = obj.val           x =..
-      obj.val = x + 1       obj.val = x + 1       obj..
+    with atomic:       with atomic:          with ..
+      x = obj.val        x = obj.val           x =..
+      obj.val = x + 1    obj.val = x + 1       obj..
 
 
 STM
@@ -134,14 +163,46 @@
 * advanced but not magic (same as databases)
 
 
-STM versus HTM
---------------
+By the way
+----------
 
-* Software versus Hardware
+* STM replaces the GIL (Global Interpreter Lock)
 
-* CPU hardware specially to avoid the high overhead
+* any existing multithreaded program runs on multiple cores
 
-* too limited for now
+
+By the way
+----------
+
+* the GIL is necessary and very hard to avoid,
+  but if you look at it like a lock around every single
+  subexpression, then it can be replaced with `with atomic` too
+
+
+So...
+-----
+
+* yes, any existing multithreaded program runs on multiple cores
+
+* yes, we solved the GIL
+
+* great
+
+
+So...
+-----
+
+* no, it would be quite hard to implement it in standard CPython
+
+* too bad for now, only in PyPy
+
+* but it would not be completely impossible
+
+
+But...
+------
+
+* but only half of the story in my opinion `:-)`
 
 
 Example 1
@@ -149,11 +210,13 @@
 
 ::
 
-  def apply_interest_rate(self):
+  def apply_interest(self):
      self.balance *= 1.05
 
+
   for account in all_accounts:
-     account.apply_interest_rate()
+     account.apply_interest()
+                                                 .
 
 
 Example 1
@@ -161,12 +224,27 @@
 
 ::
 
-  def apply_interest_rate(self):
+  def apply_interest(self):
      self.balance *= 1.05
 
+
   for account in all_accounts:
-     add_task(account.apply_interest_rate)
-  run_tasks()
+     account.apply_interest()
+     ^^^ run this loop multithreaded
+
+
+Example 1
+---------
+
+::
+
+  def apply_interest(self):
+     #with atomic: --- automatic
+        self.balance *= 1.05
+
+  for account in all_accounts:
+     add_task(account.apply_interest)
+  run_all_tasks()
 
 
 Internally
@@ -178,6 +256,8 @@
 
 * uses threads, but internally only
 
+* very simple, pure Python
+
 
 Example 2
 ---------
@@ -187,7 +267,7 @@
   def next_iteration(all_trains):
      for train in all_trains:
         start_time = ...
-        for othertrain in train.dependencies:
+        for othertrain in train.deps:
            if ...:
               start_time = ...
         train.start_time = start_time
@@ -215,37 +295,29 @@
 
 * but with `objects` instead of `records`
 
-* the transaction aborts and automatically retries
+* the transaction aborts and retries automatically
 
 
 Inevitable
 ----------
 
-* means "unavoidable"
+* "inevitable" (means "unavoidable")
 
 * handles I/O in a `with atomic`
 
 * cannot abort the transaction any more
 
 
-By the way
-----------
-
-* STM replaces the GIL
-
-* any existing multithreaded program runs on multiple cores
-
-
 Current status
 --------------
 
 * basics work, JIT compiler integration almost done
 
-* different executable called `pypy-stm`
+* different executable (`pypy-stm` instead of `pypy`)
 
 * slow-down: around 3x (in bad cases up to 10x)
 
-* speed-ups measured with 4 cores
+* real time speed-ups measured with 4 or 8 cores
 
 * Linux 64-bit only
 
@@ -258,9 +330,11 @@
 ::
 
     Detected conflict:
+      File "foo.py", line 58, in wtree
+        walk(root)
       File "foo.py", line 17, in walk
         if node.left not in seen:
-    Transaction aborted, 0.000047 seconds lost
+    Transaction aborted, 0.047 sec lost
 
 
 User feedback
@@ -273,11 +347,11 @@
     Forced inevitable:
       File "foo.py", line 19, in walk
         print >> log, logentry
-    Transaction blocked others for 0.xx seconds
+    Transaction blocked others for XX s
 
 
-Async libraries
----------------
+Asynchronous libraries
+----------------------
 
 * future work
 
@@ -287,11 +361,11 @@
 * existing Twisted apps still work, but we need to
   look at conflicts/inevitables
 
-* similar with Tornado, gevent, and so on
+* similar with Tornado, eventlib, and so on
 
 
-Async libraries
----------------
+Asynchronous libraries
+----------------------
 
 ::
 
@@ -318,6 +392,16 @@
 * reduce slow-down, port to other OS'es
 
 
+STM versus HTM
+--------------
+
+* Software versus Hardware
+
+* CPU hardware specially to avoid the high overhead (Intel Haswell processor)
+
+* too limited for now
+
+
 Under the cover
 ---------------
 
@@ -329,8 +413,8 @@
 
 * the most recent version can belong to one thread
 
-* synchronization only when a thread "steals" another thread's most
-  recent version, to make it shared
+* synchronization only at the point where one thread "steals"
+  another thread's most recent version, to make it shared
 
 * integrated with a generational garbage collector, with one
   nursery per thread
@@ -345,4 +429,8 @@
 
 * a small change for Python users
 
+* (and the GIL is gone)
+
+* this work is sponsored by crownfunding (thanks!)
+
 * `Q & A`

From noreply at buildbot.pypy.org  Mon Oct 14 18:16:24 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 18:16:24 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: start writing a blog post
Message-ID: <20131014161624.355491C02E2@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5074:963161d29f61
Date: 2013-10-14 18:16 +0200
http://bitbucket.org/pypy/extradoc/changeset/963161d29f61/

Log:	start writing a blog post

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/incremental-gc.rst
@@ -0,0 +1,52 @@
+
+Incremental Garbage Collector in PyPy
+=====================================
+
+Hello everyone.
+
+We're pleased to announce that as of today (so tomorrows nightly),
+the default PyPy comes with a GC that has much smaller pauses than yesterday.
+
+Let's start with explaining roughly what GC pauses are. In CPython each
+object has a reference count, which is incremented each time we create
+references and decremented each time we forget them. This means that objects
+are freed each time they become unreachable. That is only half of the story
+though. Consider code like this::
+
+   class A(object):
+        pass
+
+   a = A()
+   b = A()
+   a.item = b
+   b.item = a
+   del a
+   del b
+
+This creates a reference cycles. It means that while we deleted references to
+``a`` and ``b`` from the current scope, they still have a refcount of 1,
+because they point to each other, while the whole group has no reference
+from the outside. CPython employs a cyclic garbage collector that is used to
+find such cycles. It walks all objects in memory, starting from known roots
+like immortal prebuilt objects, roots on the stack etc. This solves the
+problem, but creates a noticable GC pauses when the heap becomes large and
+convoluted.
+
+PyPy has essentially only the cycle finder - it does not bother with reference
+counting, but it walks alive objects every now and then (this is a big
+simplification, PyPy's GC is much more complex than this). It also has
+a problem of GC pauses. To alleviate this problem, which is essential for
+applications like games, we started to work on incremental GC, which spreads
+the walking of objects and cleaning them across the execution time in smaller
+intervals. The work was sponsored by the Raspberry Pi foundation, started
+by Andrew Chambers and finished by Armin Rigo and Maciej Fija&#322;kowski.
+
+Benchmarks
+==========
+
+
+
+Nitty gritty details
+====================
+
+

From noreply at buildbot.pypy.org  Mon Oct 14 18:18:23 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 18:18:23 +0200 (CEST)
Subject: [pypy-commit] pypy incremental-gc: close to be merged branch
Message-ID: <20131014161823.10FA51C067F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: incremental-gc
Changeset: r67364:899396b780af
Date: 2013-10-14 18:16 +0200
http://bitbucket.org/pypy/pypy/changeset/899396b780af/

Log:	close to be merged branch


From noreply at buildbot.pypy.org  Mon Oct 14 18:18:24 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 14 Oct 2013 18:18:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal,
 arigo) A branch that implements incminimark which is
Message-ID: <20131014161824.B3C421C067F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67365:defb5119e3c6
Date: 2013-10-14 18:17 +0200
http://bitbucket.org/pypy/pypy/changeset/defb5119e3c6/

Log:	(fijal, arigo) A branch that implements incminimark which is an
	incremental version of minimark.

diff too long, truncating to 2000 out of 3052 lines

diff --git a/TODO b/TODO
new file mode 100644
--- /dev/null
+++ b/TODO
@@ -0,0 +1,24 @@
+
+
+* fix increments in major_collection_step() in the phases
+  STATE_MARKING, STATE_SWEEPING_RAWMALLOC, STATE_SWEEPING_ARENA,
+  and probably STATE_FINALIZING
+
+* 'next_major_collection_*' need to be tweaked
+
+* check the 'reserving_size' argument here and there
+
+* maybe make incremental: dealing with old_objects_with_weakrefs
+  and old_objects_with_light_finalizers and
+  deal_with_objects_with_finalizers()
+
+* REDO external_malloc(): if somebody calls this function a lot, we must
+  eventually force a full collection.
+
+* REDO card marking, starting with "card_page_indices": 128 in
+  TRANSLATION_PARAMS
+
+* write barrier: avoid the case when during sweeping we have GCFLAG_VISITED
+  on an object, so we call the slow path, but the slow path doesn't do
+  anything, and we still have GCFLAG_VISITED so we will keep calling it
+  on the same object
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -11,7 +11,7 @@
 DEFL_CLEVER_MALLOC_REMOVAL_INLINE_THRESHOLD = 32.4
 DEFL_LOW_INLINE_THRESHOLD = DEFL_INLINE_THRESHOLD / 2.0
 
-DEFL_GC = "minimark"
+DEFL_GC = "incminimark"   # XXX
 
 if sys.platform.startswith("linux"):
     DEFL_ROOTFINDER_WITHJIT = "asmgcc"
@@ -50,7 +50,7 @@
     # gc
     ChoiceOption("gc", "Garbage Collection Strategy",
                  ["boehm", "ref", "semispace", "statistics",
-                  "generation", "hybrid", "minimark", "none"],
+                  "generation", "hybrid", "minimark",'incminimark', "none"],
                   "ref", requires={
                      "ref": [("translation.rweakref", False), # XXX
                              ("translation.gctransformer", "ref")],
@@ -63,6 +63,7 @@
                      "boehm": [("translation.continuation", False),  # breaks
                                ("translation.gctransformer", "boehm")],
                      "minimark": [("translation.gctransformer", "framework")],
+                     "incminimark": [("translation.gctransformer", "framework")],
                      },
                   cmdline="--gc"),
     ChoiceOption("gctransformer", "GC transformer that is used - internal",
diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -357,7 +357,8 @@
     def _check_valid_gc(self):
         # we need the hybrid or minimark GC for rgc._make_sure_does_not_move()
         # to work.  'hybrid' could work but isn't tested with the JIT.
-        if self.gcdescr.config.translation.gc not in ('minimark',):
+        if self.gcdescr.config.translation.gc not in ('minimark',
+                                                      'incminimark'):
             raise NotImplementedError("--gc=%s not implemented with the JIT" %
                                       (self.gcdescr.config.translation.gc,))
 
diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
--- a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
+++ b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py
@@ -113,6 +113,7 @@
 
 
 class BaseFrameworkTests(object):
+    gc = DEFL_GC
 
     def setup_class(cls):
         funcs = []
@@ -162,7 +163,7 @@
         OLD_DEBUG = GcLLDescr_framework.DEBUG
         try:
             GcLLDescr_framework.DEBUG = True
-            cls.cbuilder = compile(get_entry(allfuncs), DEFL_GC,
+            cls.cbuilder = compile(get_entry(allfuncs), cls.gc,
                                    gcrootfinder=cls.gcrootfinder, jit=True,
                                    thread=True)
         finally:
diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py
--- a/rpython/jit/backend/x86/test/test_zrpy_gc.py
+++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py
@@ -3,3 +3,4 @@
 
 class TestShadowStack(CompileFrameworkTests):
     gcrootfinder = "shadowstack"
+    gc = "incminimark"
diff --git a/rpython/jit/metainterp/gc.py b/rpython/jit/metainterp/gc.py
--- a/rpython/jit/metainterp/gc.py
+++ b/rpython/jit/metainterp/gc.py
@@ -25,6 +25,9 @@
 class GC_minimark(GcDescription):
     malloc_zero_filled = True
 
+class GC_incminimark(GcDescription):
+    malloc_zero_filled = True
+
 
 def get_description(config):
     name = config.translation.gc
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -430,6 +430,7 @@
                "generation": "generation.GenerationGC",
                "hybrid": "hybrid.HybridGC",
                "minimark" : "minimark.MiniMarkGC",
+               "incminimark" : "incminimark.IncrementalMiniMarkGC",
                }
     try:
         modulename, classname = classes[config.translation.gc].split('.')
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/incminimark.py
@@ -0,0 +1,2292 @@
+"""Incremental version of the MiniMark GC.
+
+Environment variables can be used to fine-tune the following parameters:
+
+ PYPY_GC_NURSERY         The nursery size.  Defaults to 1/2 of your cache or
+                         '4M'.  Small values
+                         (like 1 or 1KB) are useful for debugging.
+
+ PYPY_GC_NURSERY_CLEANUP The interval at which nursery is cleaned up. Must
+                         be smaller than the nursery size and bigger than the
+                         biggest object we can allotate in the nursery.
+
+ PYPY_GC_INCREMENT_STEP  The size of memory marked during the marking step.
+                         Default is size of nursery * 2. If you mark it too high
+                         your GC is not incremental at all. The minimum is set
+                         to size that survives minor collection * 1.5 so we
+                         reclaim anything all the time.
+
+ PYPY_GC_MAJOR_COLLECT   Major collection memory factor.  Default is '1.82',
+                         which means trigger a major collection when the
+                         memory consumed equals 1.82 times the memory
+                         really used at the end of the previous major
+                         collection.
+
+ PYPY_GC_GROWTH          Major collection threshold's max growth rate.
+                         Default is '1.4'.  Useful to collect more often
+                         than normally on sudden memory growth, e.g. when
+                         there is a temporary peak in memory usage.
+
+ PYPY_GC_MAX             The max heap size.  If coming near this limit, it
+                         will first collect more often, then raise an
+                         RPython MemoryError, and if that is not enough,
+                         crash the program with a fatal error.  Try values
+                         like '1.6GB'.
+
+ PYPY_GC_MAX_DELTA       The major collection threshold will never be set
+                         to more than PYPY_GC_MAX_DELTA the amount really
+                         used after a collection.  Defaults to 1/8th of the
+                         total RAM size (which is constrained to be at most
+                         2/3/4GB on 32-bit systems).  Try values like '200MB'.
+
+ PYPY_GC_MIN             Don't collect while the memory size is below this
+                         limit.  Useful to avoid spending all the time in
+                         the GC in very small programs.  Defaults to 8
+                         times the nursery.
+
+ PYPY_GC_DEBUG           Enable extra checks around collections that are
+                         too slow for normal use.  Values are 0 (off),
+                         1 (on major collections) or 2 (also on minor
+                         collections).
+"""
+# XXX Should find a way to bound the major collection threshold by the
+# XXX total addressable size.  Maybe by keeping some minimarkpage arenas
+# XXX pre-reserved, enough for a few nursery collections?  What about
+# XXX raw-malloced memory?
+import sys
+from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage
+from rpython.memory.gc.base import GCBase, MovingGCBase
+from rpython.memory.gc import env
+from rpython.memory.support import mangle_hash
+from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint
+from rpython.rlib.rarithmetic import LONG_BIT_SHIFT
+from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
+from rpython.rlib.objectmodel import specialize
+
+
+#
+# Handles the objects in 2 generations:
+#
+#  * young objects: allocated in the nursery if they are not too large, or
+#    raw-malloced otherwise.  The nursery is a fixed-size memory buffer of
+#    4MB by default.  When full, we do a minor collection;
+#    the surviving objects from the nursery are moved outside, and the
+#    non-surviving raw-malloced objects are freed.  All surviving objects
+#    become old.
+#
+#  * old objects: never move again.  These objects are either allocated by
+#    minimarkpage.py (if they are small), or raw-malloced (if they are not
+#    small).  Collected by regular mark-n-sweep during major collections.
+#
+
+WORD = LONG_BIT // 8
+NULL = llmemory.NULL
+
+first_gcflag = 1 << (LONG_BIT//2)
+
+# The following flag is set on objects if we need to do something to
+# track the young pointers that it might contain.  The flag is not set
+# on young objects (unless they are large arrays, see below), and we
+# simply assume that any young object can point to any other young object.
+# For old and prebuilt objects, the flag is usually set, and is cleared
+# when we write any pointer to it.  For large arrays with
+# GCFLAG_HAS_CARDS, we rely on card marking to track where the
+# young pointers are; the flag GCFLAG_TRACK_YOUNG_PTRS is set in this
+# case too, to speed up the write barrier.
+GCFLAG_TRACK_YOUNG_PTRS = first_gcflag << 0
+
+# The following flag is set on some prebuilt objects.  The flag is set
+# unless the object is already listed in 'prebuilt_root_objects'.
+# When a pointer is written inside an object with GCFLAG_NO_HEAP_PTRS
+# set, the write_barrier clears the flag and adds the object to
+# 'prebuilt_root_objects'.
+GCFLAG_NO_HEAP_PTRS = first_gcflag << 1
+
+# The following flag is set on surviving objects during a major collection.
+GCFLAG_VISITED      = first_gcflag << 2
+
+# The following flag is set on nursery objects of which we asked the id
+# or the identityhash.  It means that a space of the size of the object
+# has already been allocated in the nonmovable part.  The same flag is
+# abused to mark prebuilt objects whose hash has been taken during
+# translation and is statically recorded.
+GCFLAG_HAS_SHADOW   = first_gcflag << 3
+
+# The following flag is set temporarily on some objects during a major
+# collection.  See pypy/doc/discussion/finalizer-order.txt
+GCFLAG_FINALIZATION_ORDERING = first_gcflag << 4
+
+# This flag is reserved for RPython.
+GCFLAG_EXTRA        = first_gcflag << 5
+
+# The following flag is set on externally raw_malloc'ed arrays of pointers.
+# They are allocated with some extra space in front of them for a bitfield,
+# one bit per 'card_page_indices' indices.
+GCFLAG_HAS_CARDS    = first_gcflag << 6
+GCFLAG_CARDS_SET    = first_gcflag << 7     # <- at least one card bit is set
+# note that GCFLAG_CARDS_SET is the most significant bit of a byte:
+# this is required for the JIT (x86)
+
+# The following flag is set on surviving raw-malloced young objects during
+# a minor collection.
+GCFLAG_VISITED_RMY   = first_gcflag << 8
+
+_GCFLAG_FIRST_UNUSED = first_gcflag << 9    # the first unused bit
+
+
+# States for the incremental GC
+
+# The scanning phase, next step call will scan the current roots
+# This state must complete in a single step
+STATE_SCANNING = 0
+
+# The marking phase. We walk the list 'objects_to_trace' of all gray objects
+# and mark all of the things they point to gray. This step lasts until there
+# are no more gray objects.
+STATE_MARKING = 1
+
+# here we kill all the unvisited objects
+STATE_SWEEPING = 2
+
+# here we call all the finalizers
+STATE_FINALIZING = 3
+
+GC_STATES = ['SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING']
+
+
+FORWARDSTUB = lltype.GcStruct('forwarding_stub',
+                              ('forw', llmemory.Address))
+FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB)
+NURSARRAY = lltype.Array(llmemory.Address)
+
+# ____________________________________________________________
+
+class IncrementalMiniMarkGC(MovingGCBase):
+    _alloc_flavor_ = "raw"
+    inline_simple_malloc = True
+    inline_simple_malloc_varsize = True
+    needs_write_barrier = True
+    prebuilt_gc_objects_are_static_roots = False
+    malloc_zero_filled = True    # xxx experiment with False
+    gcflag_extra = GCFLAG_EXTRA
+
+    # All objects start with a HDR, i.e. with a field 'tid' which contains
+    # a word.  This word is divided in two halves: the lower half contains
+    # the typeid, and the upper half contains various flags, as defined
+    # by GCFLAG_xxx above.
+    HDR = lltype.Struct('header', ('tid', lltype.Signed))
+    typeid_is_in_field = 'tid'
+    withhash_flag_is_in_field = 'tid', GCFLAG_HAS_SHADOW
+    # ^^^ prebuilt objects may have the flag GCFLAG_HAS_SHADOW;
+    #     then they are one word longer, the extra word storing the hash.
+
+
+    # During a minor collection, the objects in the nursery that are
+    # moved outside are changed in-place: their header is replaced with
+    # the value -42, and the following word is set to the address of
+    # where the object was moved.  This means that all objects in the
+    # nursery need to be at least 2 words long, but objects outside the
+    # nursery don't need to.
+    minimal_size_in_nursery = (
+        llmemory.sizeof(HDR) + llmemory.sizeof(llmemory.Address))
+
+
+    TRANSLATION_PARAMS = {
+        # Automatically adjust the size of the nursery and the
+        # 'major_collection_threshold' from the environment.
+        # See docstring at the start of the file.
+        "read_from_env": True,
+
+        # The size of the nursery.  Note that this is only used as a
+        # fall-back number.
+        "nursery_size": 896*1024,
+
+        # The system page size.  Like obmalloc.c, we assume that it is 4K
+        # for 32-bit systems; unlike obmalloc.c, we assume that it is 8K
+        # for 64-bit systems, for consistent results.
+        "page_size": 1024*WORD,
+
+        # The size of an arena.  Arenas are groups of pages allocated
+        # together.
+        "arena_size": 65536*WORD,
+
+        # The maximum size of an object allocated compactly.  All objects
+        # that are larger are just allocated with raw_malloc().  Note that
+        # the size limit for being first allocated in the nursery is much
+        # larger; see below.
+        "small_request_threshold": 35*WORD,
+
+        # Full collection threshold: after a major collection, we record
+        # the total size consumed; and after every minor collection, if the
+        # total size is now more than 'major_collection_threshold' times,
+        # we trigger the next major collection.
+        "major_collection_threshold": 1.82,
+
+        # Threshold to avoid that the total heap size grows by a factor of
+        # major_collection_threshold at every collection: it can only
+        # grow at most by the following factor from one collection to the
+        # next.  Used e.g. when there is a sudden, temporary peak in memory
+        # usage; this avoids that the upper bound grows too fast.
+        "growth_rate_max": 1.4,
+
+        # The number of array indices that are mapped to a single bit in
+        # write_barrier_from_array().  Must be a power of two.  The default
+        # value of 128 means that card pages are 512 bytes (1024 on 64-bits)
+        # in regular arrays of pointers; more in arrays whose items are
+        # larger.  A value of 0 disables card marking.
+        "card_page_indices": 128,
+
+        # Objects whose total size is at least 'large_object' bytes are
+        # allocated out of the nursery immediately, as old objects.  The
+        # minimal allocated size of the nursery is 2x the following
+        # number (by default, at least 132KB on 32-bit and 264KB on 64-bit).
+        "large_object": (16384+512)*WORD,
+
+        # This is the chunk that we cleanup in the nursery. The point is
+        # to avoid having to trash all the caches just to zero the nursery,
+        # so we trade it by cleaning it bit-by-bit, as we progress through
+        # nursery. Has to fit at least one large object
+        "nursery_cleanup": 32768 * WORD,
+        }
+
+    def __init__(self, config,
+                 read_from_env=False,
+                 nursery_size=32*WORD,
+                 nursery_cleanup=9*WORD,
+                 page_size=16*WORD,
+                 arena_size=64*WORD,
+                 small_request_threshold=5*WORD,
+                 major_collection_threshold=2.5,
+                 growth_rate_max=2.5,   # for tests
+                 card_page_indices=0,
+                 large_object=8*WORD,
+                 ArenaCollectionClass=None,
+                 **kwds):
+        MovingGCBase.__init__(self, config, **kwds)
+        assert small_request_threshold % WORD == 0
+        self.read_from_env = read_from_env
+        self.nursery_size = nursery_size
+        self.nursery_cleanup = nursery_cleanup
+        self.small_request_threshold = small_request_threshold
+        self.major_collection_threshold = major_collection_threshold
+        self.growth_rate_max = growth_rate_max
+        self.num_major_collects = 0
+        self.min_heap_size = 0.0
+        self.max_heap_size = 0.0
+        self.max_heap_size_already_raised = False
+        self.max_delta = float(r_uint(-1))
+        #
+        self.card_page_indices = card_page_indices
+        if self.card_page_indices > 0:
+            self.card_page_shift = 0
+            while (1 << self.card_page_shift) < self.card_page_indices:
+                self.card_page_shift += 1
+        #
+        # 'large_object' limit how big objects can be in the nursery, so
+        # it gives a lower bound on the allowed size of the nursery.
+        self.nonlarge_max = large_object - 1
+        #
+        self.nursery      = NULL
+        self.nursery_free = NULL
+        self.nursery_top  = NULL
+        self.nursery_real_top = NULL
+        self.debug_tiny_nursery = -1
+        self.debug_rotating_nurseries = lltype.nullptr(NURSARRAY)
+        self.extra_threshold = 0
+        #
+        # The ArenaCollection() handles the nonmovable objects allocation.
+        if ArenaCollectionClass is None:
+            from rpython.memory.gc import minimarkpage
+            ArenaCollectionClass = minimarkpage.ArenaCollection
+        self.ac = ArenaCollectionClass(arena_size, page_size,
+                                       small_request_threshold)
+        #
+        # Used by minor collection: a list of (mostly non-young) objects that
+        # (may) contain a pointer to a young object.  Populated by
+        # the write barrier: when we clear GCFLAG_TRACK_YOUNG_PTRS, we
+        # add it to this list.
+        # Note that young array objects may (by temporary "mistake") be added
+        # to this list, but will be removed again at the start of the next
+        # minor collection.
+        self.old_objects_pointing_to_young = self.AddressStack()
+        #
+        # Similar to 'old_objects_pointing_to_young', but lists objects
+        # that have the GCFLAG_CARDS_SET bit.  For large arrays.  Note
+        # that it is possible for an object to be listed both in here
+        # and in 'old_objects_pointing_to_young', in which case we
+        # should just clear the cards and trace it fully, as usual.
+        # Note also that young array objects are never listed here.
+        self.old_objects_with_cards_set = self.AddressStack()
+        #
+        # A list of all prebuilt GC objects that contain pointers to the heap
+        self.prebuilt_root_objects = self.AddressStack()
+        #
+        self._init_writebarrier_logic()
+
+
+    def setup(self):
+        """Called at run-time to initialize the GC."""
+        #
+        # Hack: MovingGCBase.setup() sets up stuff related to id(), which
+        # we implement differently anyway.  So directly call GCBase.setup().
+        GCBase.setup(self)
+        #
+        # Two lists of all raw_malloced objects (the objects too large)
+        self.young_rawmalloced_objects = self.null_address_dict()
+        self.old_rawmalloced_objects = self.AddressStack()
+        self.raw_malloc_might_sweep = self.AddressStack()
+        self.rawmalloced_total_size = r_uint(0)
+
+        self.gc_state = STATE_SCANNING
+        #
+        # A list of all objects with finalizers (these are never young).
+        self.objects_with_finalizers = self.AddressDeque()
+        self.young_objects_with_light_finalizers = self.AddressStack()
+        self.old_objects_with_light_finalizers = self.AddressStack()
+        #
+        # Two lists of the objects with weakrefs.  No weakref can be an
+        # old object weakly pointing to a young object: indeed, weakrefs
+        # are immutable so they cannot point to an object that was
+        # created after it.
+        self.young_objects_with_weakrefs = self.AddressStack()
+        self.old_objects_with_weakrefs = self.AddressStack()
+        #
+        # Support for id and identityhash: map nursery objects with
+        # GCFLAG_HAS_SHADOW to their future location at the next
+        # minor collection.
+        self.nursery_objects_shadows = self.AddressDict()
+        #
+        # Allocate a nursery.  In case of auto_nursery_size, start by
+        # allocating a very small nursery, enough to do things like look
+        # up the env var, which requires the GC; and then really
+        # allocate the nursery of the final size.
+        if not self.read_from_env:
+            self.allocate_nursery()
+            self.gc_increment_step = self.nursery_size * 4
+        else:
+            #
+            defaultsize = self.nursery_size
+            minsize = 2 * (self.nonlarge_max + 1)
+            self.nursery_size = minsize
+            self.allocate_nursery()
+            #
+            # From there on, the GC is fully initialized and the code
+            # below can use it
+            newsize = env.read_from_env('PYPY_GC_NURSERY')
+            # PYPY_GC_NURSERY=smallvalue means that minor collects occur
+            # very frequently; the extreme case is PYPY_GC_NURSERY=1, which
+            # forces a minor collect for every malloc.  Useful to debug
+            # external factors, like trackgcroot or the handling of the write
+            # barrier.  Implemented by still using 'minsize' for the nursery
+            # size (needed to handle mallocs just below 'large_objects') but
+            # hacking at the current nursery position in collect_and_reserve().
+            if newsize <= 0:
+                newsize = env.estimate_best_nursery_size()
+                if newsize <= 0:
+                    newsize = defaultsize
+            if newsize < minsize:
+                self.debug_tiny_nursery = newsize & ~(WORD-1)
+                newsize = minsize
+
+            nurs_cleanup = env.read_from_env('PYPY_GC_NURSERY_CLEANUP')
+            if nurs_cleanup > 0:
+                self.nursery_cleanup = nurs_cleanup
+            #
+            major_coll = env.read_float_from_env('PYPY_GC_MAJOR_COLLECT')
+            if major_coll > 1.0:
+                self.major_collection_threshold = major_coll
+            #
+            growth = env.read_float_from_env('PYPY_GC_GROWTH')
+            if growth > 1.0:
+                self.growth_rate_max = growth
+            #
+            min_heap_size = env.read_uint_from_env('PYPY_GC_MIN')
+            if min_heap_size > 0:
+                self.min_heap_size = float(min_heap_size)
+            else:
+                # defaults to 8 times the nursery
+                self.min_heap_size = newsize * 8
+            #
+            max_heap_size = env.read_uint_from_env('PYPY_GC_MAX')
+            if max_heap_size > 0:
+                self.max_heap_size = float(max_heap_size)
+            #
+            max_delta = env.read_uint_from_env('PYPY_GC_MAX_DELTA')
+            if max_delta > 0:
+                self.max_delta = float(max_delta)
+            else:
+                self.max_delta = 0.125 * env.get_total_memory()
+
+            gc_increment_step = env.read_uint_from_env('PYPY_GC_INCREMENT_STEP')
+            if gc_increment_step > 0:
+                self.gc_increment_step = gc_increment_step
+            else:
+                self.gc_increment_step = newsize * 4
+            #
+            self.minor_collection()    # to empty the nursery
+            llarena.arena_free(self.nursery)
+            self.nursery_size = newsize
+            self.allocate_nursery()
+        #
+        if self.nursery_cleanup < self.nonlarge_max + 1:
+            self.nursery_cleanup = self.nonlarge_max + 1
+        # We need exactly initial_cleanup + N*nursery_cleanup = nursery_size.
+        # We choose the value of initial_cleanup to be between 1x and 2x the
+        # value of nursery_cleanup.
+        self.initial_cleanup = self.nursery_cleanup + (
+                self.nursery_size % self.nursery_cleanup)
+        if (r_uint(self.initial_cleanup) > r_uint(self.nursery_size) or
+            self.debug_tiny_nursery >= 0):
+            self.initial_cleanup = self.nursery_size
+
+    def _nursery_memory_size(self):
+        extra = self.nonlarge_max + 1
+        return self.nursery_size + extra
+
+    def _alloc_nursery(self):
+        # the start of the nursery: we actually allocate a bit more for
+        # the nursery than really needed, to simplify pointer arithmetic
+        # in malloc_fixedsize_clear().  The few extra pages are never used
+        # anyway so it doesn't even count.
+        nursery = llarena.arena_malloc(self._nursery_memory_size(), 2)
+        if not nursery:
+            raise MemoryError("cannot allocate nursery")
+        return nursery
+
+    def allocate_nursery(self):
+        debug_start("gc-set-nursery-size")
+        debug_print("nursery size:", self.nursery_size)
+        self.nursery = self._alloc_nursery()
+        # the current position in the nursery:
+        self.nursery_free = self.nursery
+        # the end of the nursery:
+        self.nursery_top = self.nursery + self.nursery_size
+        self.nursery_real_top = self.nursery_top
+        # initialize the threshold
+        self.min_heap_size = max(self.min_heap_size, self.nursery_size *
+                                              self.major_collection_threshold)
+        # the following two values are usually equal, but during raw mallocs
+        # of arrays, next_major_collection_threshold is decremented to make
+        # the next major collection arrive earlier.
+        # See translator/c/test/test_newgc, test_nongc_attached_to_gc
+        self.next_major_collection_initial = self.min_heap_size
+        self.next_major_collection_threshold = self.min_heap_size
+        self.set_major_threshold_from(0.0)
+        ll_assert(self.extra_threshold == 0, "extra_threshold set too early")
+        self.initial_cleanup = self.nursery_size
+        debug_stop("gc-set-nursery-size")
+
+
+    def set_major_threshold_from(self, threshold, reserving_size=0):
+        # Set the next_major_collection_threshold.
+        threshold_max = (self.next_major_collection_initial *
+                         self.growth_rate_max)
+        if threshold > threshold_max:
+            threshold = threshold_max
+        #
+        threshold += reserving_size
+        if threshold < self.min_heap_size:
+            threshold = self.min_heap_size
+        #
+        if self.max_heap_size > 0.0 and threshold > self.max_heap_size:
+            threshold = self.max_heap_size
+            bounded = True
+        else:
+            bounded = False
+        #
+        self.next_major_collection_initial = threshold
+        self.next_major_collection_threshold = threshold
+        return bounded
+
+
+    def post_setup(self):
+        # set up extra stuff for PYPY_GC_DEBUG.
+        MovingGCBase.post_setup(self)
+        if self.DEBUG and llarena.has_protect:
+            # gc debug mode: allocate 23 nurseries instead of just 1,
+            # and use them alternatively, while mprotect()ing the unused
+            # ones to detect invalid access.
+            debug_start("gc-debug")
+            self.debug_rotating_nurseries = lltype.malloc(
+                NURSARRAY, 22, flavor='raw', track_allocation=False)
+            i = 0
+            while i < 22:
+                nurs = self._alloc_nursery()
+                llarena.arena_protect(nurs, self._nursery_memory_size(), True)
+                self.debug_rotating_nurseries[i] = nurs
+                i += 1
+            debug_print("allocated", len(self.debug_rotating_nurseries),
+                        "extra nurseries")
+            debug_stop("gc-debug")
+
+    def debug_rotate_nursery(self):
+        if self.debug_rotating_nurseries:
+            debug_start("gc-debug")
+            oldnurs = self.nursery
+            llarena.arena_protect(oldnurs, self._nursery_memory_size(), True)
+            #
+            newnurs = self.debug_rotating_nurseries[0]
+            i = 0
+            while i < len(self.debug_rotating_nurseries) - 1:
+                self.debug_rotating_nurseries[i] = (
+                    self.debug_rotating_nurseries[i + 1])
+                i += 1
+            self.debug_rotating_nurseries[i] = oldnurs
+            #
+            llarena.arena_protect(newnurs, self._nursery_memory_size(), False)
+            self.nursery = newnurs
+            self.nursery_top = self.nursery + self.initial_cleanup
+            self.nursery_real_top = self.nursery + self.nursery_size
+            debug_print("switching from nursery", oldnurs,
+                        "to nursery", self.nursery,
+                        "size", self.nursery_size)
+            debug_stop("gc-debug")
+
+
+    def malloc_fixedsize_clear(self, typeid, size,
+                               needs_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        totalsize = size_gc_header + size
+        rawtotalsize = raw_malloc_usage(totalsize)
+        #
+        # If the object needs a finalizer, ask for a rawmalloc.
+        # The following check should be constant-folded.
+        if needs_finalizer and not is_finalizer_light:
+            ll_assert(not contains_weakptr,
+                     "'needs_finalizer' and 'contains_weakptr' both specified")
+            obj = self.external_malloc(typeid, 0, can_make_young=False)
+            self.objects_with_finalizers.append(obj)
+        #
+        # If totalsize is greater than nonlarge_max (which should never be
+        # the case in practice), ask for a rawmalloc.  The following check
+        # should be constant-folded.
+        elif rawtotalsize > self.nonlarge_max:
+            ll_assert(not contains_weakptr,
+                      "'contains_weakptr' specified for a large object")
+            obj = self.external_malloc(typeid, 0)
+            #
+        else:
+            # If totalsize is smaller than minimal_size_in_nursery, round it
+            # up.  The following check should also be constant-folded.
+            min_size = raw_malloc_usage(self.minimal_size_in_nursery)
+            if rawtotalsize < min_size:
+                totalsize = rawtotalsize = min_size
+            #
+            # Get the memory from the nursery.  If there is not enough space
+            # there, do a collect first.
+            result = self.nursery_free
+            self.nursery_free = result + totalsize
+            if self.nursery_free > self.nursery_top:
+                result = self.collect_and_reserve(result, totalsize)
+            #
+            # Build the object.
+            llarena.arena_reserve(result, totalsize)
+            obj = result + size_gc_header
+            if is_finalizer_light:
+                self.young_objects_with_light_finalizers.append(obj)
+            self.init_gc_object(result, typeid, flags=0)
+            #
+            # If it is a weakref, record it (check constant-folded).
+            if contains_weakptr:
+                self.young_objects_with_weakrefs.append(obj)
+        #
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+
+
+    def malloc_varsize_clear(self, typeid, length, size, itemsize,
+                             offset_to_length):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        nonvarsize = size_gc_header + size
+        #
+        # Compute the maximal length that makes the object still
+        # below 'nonlarge_max'.  All the following logic is usually
+        # constant-folded because self.nonlarge_max, size and itemsize
+        # are all constants (the arguments are constant due to
+        # inlining).
+        maxsize = self.nonlarge_max - raw_malloc_usage(nonvarsize)
+        if maxsize < 0:
+            toobig = r_uint(0)    # the nonvarsize alone is too big
+        elif raw_malloc_usage(itemsize):
+            toobig = r_uint(maxsize // raw_malloc_usage(itemsize)) + 1
+        else:
+            toobig = r_uint(sys.maxint) + 1
+
+        if r_uint(length) >= r_uint(toobig):
+            #
+            # If the total size of the object would be larger than
+            # 'nonlarge_max', then allocate it externally.  We also
+            # go there if 'length' is actually negative.
+            obj = self.external_malloc(typeid, length)
+            #
+        else:
+            # With the above checks we know now that totalsize cannot be more
+            # than 'nonlarge_max'; in particular, the + and * cannot overflow.
+            totalsize = nonvarsize + itemsize * length
+            totalsize = llarena.round_up_for_allocation(totalsize)
+            #
+            # 'totalsize' should contain at least the GC header and
+            # the length word, so it should never be smaller than
+            # 'minimal_size_in_nursery'
+            ll_assert(raw_malloc_usage(totalsize) >=
+                      raw_malloc_usage(self.minimal_size_in_nursery),
+                      "malloc_varsize_clear(): totalsize < minimalsize")
+            #
+            # Get the memory from the nursery.  If there is not enough space
+            # there, do a collect first.
+            result = self.nursery_free
+            self.nursery_free = result + totalsize
+            if self.nursery_free > self.nursery_top:
+                result = self.collect_and_reserve(result, totalsize)
+            #
+            # Build the object.
+            llarena.arena_reserve(result, totalsize)
+            self.init_gc_object(result, typeid, flags=0)
+            #
+            # Set the length and return the object.
+            obj = result + size_gc_header
+            (obj + offset_to_length).signed[0] = length
+        #
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+
+
+    def collect(self, gen=1):
+        """Do a minor (gen=0) or full major (gen>0) collection."""
+        if gen > 0:
+            self.minor_and_major_collection()
+        else:
+            self.minor_collection()
+
+    def move_nursery_top(self, totalsize):
+        size = self.nursery_cleanup
+        ll_assert(self.nursery_real_top - self.nursery_top >= size,
+            "nursery_cleanup not a divisor of nursery_size - initial_cleanup")
+        ll_assert(llmemory.raw_malloc_usage(totalsize) <= size,
+            "totalsize > nursery_cleanup")
+        llarena.arena_reset(self.nursery_top, size, 2)
+        self.nursery_top += size
+    move_nursery_top._always_inline_ = True
+
+    def collect_and_reserve(self, prev_result, totalsize):
+        """To call when nursery_free overflows nursery_top.
+        First check if the nursery_top is the real top, otherwise we
+        can just move the top of one cleanup and continue
+
+        Do a minor collection, and possibly also a major collection,
+        and finally reserve 'totalsize' bytes at the start of the
+        now-empty nursery.
+        """
+        if self.nursery_top < self.nursery_real_top:
+            self.move_nursery_top(totalsize)
+            return prev_result
+        self.minor_collection()
+        #
+        # If the gc_state is not STATE_SCANNING, we're in the middle of
+        # an incremental major collection.  In this case, always progress
+        # one step.  If the gc_state is STATE_SCANNING, wait until there
+        # is too much garbage before starting the next major collection.
+        if (self.gc_state != STATE_SCANNING or
+                    self.get_total_memory_used() >
+                    self.next_major_collection_threshold):
+            self.major_collection_step()
+            #
+            # The nursery might not be empty now, because of
+            # execute_finalizers().  If it is almost full again,
+            # we need to fix it with another call to minor_collection().
+            if self.nursery_free + totalsize > self.nursery_top:
+                #
+                if self.nursery_free + totalsize > self.nursery_real_top:
+                    self.minor_collection()
+                    # then the nursery is empty
+                else:
+                    # we just need to clean up a bit more of the nursery
+                    self.move_nursery_top(totalsize)
+        #
+        result = self.nursery_free
+        self.nursery_free = result + totalsize
+        ll_assert(self.nursery_free <= self.nursery_top, "nursery overflow")
+        #
+        if self.debug_tiny_nursery >= 0:   # for debugging
+            if self.nursery_top - self.nursery_free > self.debug_tiny_nursery:
+                self.nursery_free = self.nursery_top - self.debug_tiny_nursery
+        #
+        return result
+    collect_and_reserve._dont_inline_ = True
+
+
+    def external_malloc(self, typeid, length, can_make_young=True):
+        """Allocate a large object using the ArenaCollection or
+        raw_malloc(), possibly as an object with card marking enabled,
+        if it has gc pointers in its var-sized part.  'length' should be
+        specified as 0 if the object is not varsized.  The returned
+        object is fully initialized and zero-filled."""
+        #
+        # Here we really need a valid 'typeid', not 0 (as the JIT might
+        # try to send us if there is still a bug).
+        ll_assert(bool(self.combine(typeid, 0)),
+                  "external_malloc: typeid == 0")
+        #
+        # Compute the total size, carefully checking for overflows.
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        nonvarsize = size_gc_header + self.fixed_size(typeid)
+        if length == 0:
+            # this includes the case of fixed-size objects, for which we
+            # should not even ask for the varsize_item_sizes().
+            totalsize = nonvarsize
+        elif length > 0:
+            # var-sized allocation with at least one item
+            itemsize = self.varsize_item_sizes(typeid)
+            try:
+                varsize = ovfcheck(itemsize * length)
+                totalsize = ovfcheck(nonvarsize + varsize)
+            except OverflowError:
+                raise MemoryError
+        else:
+            # negative length!  This likely comes from an overflow
+            # earlier.  We will just raise MemoryError here.
+            raise MemoryError
+        #
+        # If somebody calls this function a lot, we must eventually
+        # force a full collection.  XXX make this more incremental!
+        if (float(self.get_total_memory_used()) + raw_malloc_usage(totalsize) >
+                self.next_major_collection_threshold):
+            self.gc_step_until(STATE_SWEEPING)
+            self.gc_step_until(STATE_FINALIZING, raw_malloc_usage(totalsize))
+        #
+        # Check if the object would fit in the ArenaCollection.
+        if raw_malloc_usage(totalsize) <= self.small_request_threshold:
+            #
+            # Yes.  Round up 'totalsize' (it cannot overflow and it
+            # must remain <= self.small_request_threshold.)
+            totalsize = llarena.round_up_for_allocation(totalsize)
+            ll_assert(raw_malloc_usage(totalsize) <=
+                      self.small_request_threshold,
+                      "rounding up made totalsize > small_request_threshold")
+            #
+            # Allocate from the ArenaCollection and clear the memory returned.
+            result = self.ac.malloc(totalsize)
+            llmemory.raw_memclear(result, totalsize)
+            #
+            # An object allocated from ArenaCollection is always old, even
+            # if 'can_make_young'.  The interesting case of 'can_make_young'
+            # is for large objects, bigger than the 'large_objects' threshold,
+            # which are raw-malloced but still young.
+            extra_flags = GCFLAG_TRACK_YOUNG_PTRS
+            #
+        else:
+            # No, so proceed to allocate it externally with raw_malloc().
+            # Check if we need to introduce the card marker bits area.
+            if (self.card_page_indices <= 0  # <- this check is constant-folded
+                or not self.has_gcptr_in_varsize(typeid) or
+                raw_malloc_usage(totalsize) <= self.nonlarge_max):
+                #
+                # In these cases, we don't want a card marker bits area.
+                # This case also includes all fixed-size objects.
+                cardheadersize = 0
+                extra_flags = 0
+                #
+            else:
+                # Reserve N extra words containing card bits before the object.
+                extra_words = self.card_marking_words_for_length(length)
+                cardheadersize = WORD * extra_words
+                extra_flags = GCFLAG_HAS_CARDS | GCFLAG_TRACK_YOUNG_PTRS
+                # if 'can_make_young', then we also immediately set
+                # GCFLAG_CARDS_SET, but without adding the object to
+                # 'old_objects_with_cards_set'.  In this way it should
+                # never be added to that list as long as it is young.
+                if can_make_young:
+                    extra_flags |= GCFLAG_CARDS_SET
+            #
+            # Detect very rare cases of overflows
+            if raw_malloc_usage(totalsize) > (sys.maxint - (WORD-1)
+                                              - cardheadersize):
+                raise MemoryError("rare case of overflow")
+            #
+            # Now we know that the following computations cannot overflow.
+            # Note that round_up_for_allocation() is also needed to get the
+            # correct number added to 'rawmalloced_total_size'.
+            allocsize = (cardheadersize + raw_malloc_usage(
+                            llarena.round_up_for_allocation(totalsize)))
+            #
+            # Allocate the object using arena_malloc(), which we assume here
+            # is just the same as raw_malloc(), but allows the extra
+            # flexibility of saying that we have extra words in the header.
+            # The memory returned is cleared by a raw_memclear().
+            arena = llarena.arena_malloc(allocsize, 2)
+            if not arena:
+                raise MemoryError("cannot allocate large object")
+            #
+            # Reserve the card mark bits as a list of single bytes
+            # (the loop is empty in C).
+            i = 0
+            while i < cardheadersize:
+                llarena.arena_reserve(arena + i, llmemory.sizeof(lltype.Char))
+                i += 1
+            #
+            # Reserve the actual object.  (This is also a no-op in C).
+            result = arena + cardheadersize
+            llarena.arena_reserve(result, totalsize)
+            #
+            # Record the newly allocated object and its full malloced size.
+            # The object is young or old depending on the argument.
+            self.rawmalloced_total_size += r_uint(allocsize)
+            if can_make_young:
+                if not self.young_rawmalloced_objects:
+                    self.young_rawmalloced_objects = self.AddressDict()
+                self.young_rawmalloced_objects.add(result + size_gc_header)
+            else:
+                self.old_rawmalloced_objects.append(result + size_gc_header)
+                extra_flags |= GCFLAG_TRACK_YOUNG_PTRS
+        #
+        # Common code to fill the header and length of the object.
+        self.init_gc_object(result, typeid, extra_flags)
+        if self.is_varsize(typeid):
+            offset_to_length = self.varsize_offset_to_length(typeid)
+            (result + size_gc_header + offset_to_length).signed[0] = length
+        return result + size_gc_header
+
+
+    # ----------
+    # Other functions in the GC API
+
+    def set_max_heap_size(self, size):
+        self.max_heap_size = float(size)
+        if self.max_heap_size > 0.0:
+            if self.max_heap_size < self.next_major_collection_initial:
+                self.next_major_collection_initial = self.max_heap_size
+            if self.max_heap_size < self.next_major_collection_threshold:
+                self.next_major_collection_threshold = self.max_heap_size
+
+    def raw_malloc_memory_pressure(self, sizehint):
+        self.next_major_collection_threshold -= sizehint
+        if self.next_major_collection_threshold < 0:
+            # cannot trigger a full collection now, but we can ensure
+            # that one will occur very soon
+            self.nursery_top = self.nursery_real_top
+            self.nursery_free = self.nursery_real_top
+
+    def can_malloc_nonmovable(self):
+        return True
+
+    def can_optimize_clean_setarrayitems(self):
+        if self.card_page_indices > 0:
+            return False
+        return MovingGCBase.can_optimize_clean_setarrayitems(self)
+
+    def can_move(self, obj):
+        """Overrides the parent can_move()."""
+        return self.is_in_nursery(obj)
+
+
+    def shrink_array(self, obj, smallerlength):
+        #
+        # Only objects in the nursery can be "resized".  Resizing them
+        # means recording that they have a smaller size, so that when
+        # moved out of the nursery, they will consume less memory.
+        # In particular, an array with GCFLAG_HAS_CARDS is never resized.
+        # Also, a nursery object with GCFLAG_HAS_SHADOW is not resized
+        # either, as this would potentially loose part of the memory in
+        # the already-allocated shadow.
+        if not self.is_in_nursery(obj):
+            return False
+        if self.header(obj).tid & GCFLAG_HAS_SHADOW:
+            return False
+        #
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        typeid = self.get_type_id(obj)
+        totalsmallersize = (
+            size_gc_header + self.fixed_size(typeid) +
+            self.varsize_item_sizes(typeid) * smallerlength)
+        llarena.arena_shrink_obj(obj - size_gc_header, totalsmallersize)
+        #
+        offset_to_length = self.varsize_offset_to_length(typeid)
+        (obj + offset_to_length).signed[0] = smallerlength
+        return True
+
+
+    def malloc_fixedsize_nonmovable(self, typeid):
+        obj = self.external_malloc(typeid, 0)
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+
+    def malloc_varsize_nonmovable(self, typeid, length):
+        obj = self.external_malloc(typeid, length)
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+
+    def malloc_nonmovable(self, typeid, length, zero):
+        # helper for testing, same as GCBase.malloc
+        return self.external_malloc(typeid, length or 0)    # None -> 0
+
+
+    # ----------
+    # Simple helpers
+
+    def get_type_id(self, obj):
+        tid = self.header(obj).tid
+        return llop.extract_ushort(llgroup.HALFWORD, tid)
+
+    def combine(self, typeid16, flags):
+        return llop.combine_ushort(lltype.Signed, typeid16, flags)
+
+    def init_gc_object(self, addr, typeid16, flags=0):
+        # The default 'flags' is zero.  The flags GCFLAG_NO_xxx_PTRS
+        # have been chosen to allow 'flags' to be zero in the common
+        # case (hence the 'NO' in their name).
+        hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+        hdr.tid = self.combine(typeid16, flags)
+
+    def init_gc_object_immortal(self, addr, typeid16, flags=0):
+        # For prebuilt GC objects, the flags must contain
+        # GCFLAG_NO_xxx_PTRS, at least initially.
+        flags |= GCFLAG_NO_HEAP_PTRS | GCFLAG_TRACK_YOUNG_PTRS
+        self.init_gc_object(addr, typeid16, flags)
+
+    def is_in_nursery(self, addr):
+        ll_assert(llmemory.cast_adr_to_int(addr) & 1 == 0,
+                  "odd-valued (i.e. tagged) pointer unexpected here")
+        return self.nursery <= addr < self.nursery_real_top
+
+    def appears_to_be_young(self, addr):
+        # "is a valid addr to a young object?"
+        # but it's ok to occasionally return True accidentally.
+        # Maybe the best implementation would be a bloom filter
+        # of some kind instead of the dictionary lookup that is
+        # sometimes done below.  But the expected common answer
+        # is "Yes" because addr points to the nursery, so it may
+        # not be useful to optimize the other case too much.
+        #
+        # First, if 'addr' appears to be a pointer to some place within
+        # the nursery, return True
+        if not self.translated_to_c:
+            # When non-translated, filter out tagged pointers explicitly.
+            # When translated, it may occasionally give a wrong answer
+            # of True if 'addr' is a tagged pointer with just the wrong value.
+            if not self.is_valid_gc_object(addr):
+                return False
+
+        if self.nursery <= addr < self.nursery_real_top:
+            return True      # addr is in the nursery
+        #
+        # Else, it may be in the set 'young_rawmalloced_objects'
+        return (bool(self.young_rawmalloced_objects) and
+                self.young_rawmalloced_objects.contains(addr))
+    appears_to_be_young._always_inline_ = True
+
+    def debug_is_old_object(self, addr):
+        return (self.is_valid_gc_object(addr)
+                and not self.appears_to_be_young(addr))
+
+    def is_forwarded(self, obj):
+        """Returns True if the nursery obj is marked as forwarded.
+        Implemented a bit obscurely by checking an unrelated flag
+        that can never be set on a young object -- except if tid == -42.
+        """
+        assert self.is_in_nursery(obj)
+        tid = self.header(obj).tid
+        result = (tid & GCFLAG_FINALIZATION_ORDERING != 0)
+        if result:
+            ll_assert(tid == -42, "bogus header for young obj")
+        else:
+            ll_assert(bool(tid), "bogus header (1)")
+            ll_assert(tid & -_GCFLAG_FIRST_UNUSED == 0, "bogus header (2)")
+        return result
+
+    def get_forwarding_address(self, obj):
+        return llmemory.cast_adr_to_ptr(obj, FORWARDSTUBPTR).forw
+
+    def get_possibly_forwarded_type_id(self, obj):
+        if self.is_in_nursery(obj) and self.is_forwarded(obj):
+            obj = self.get_forwarding_address(obj)
+        return self.get_type_id(obj)
+
+    def get_total_memory_used(self):
+        """Return the total memory used, not counting any object in the
+        nursery: only objects in the ArenaCollection or raw-malloced.
+        """
+        return self.ac.total_memory_used + self.rawmalloced_total_size
+
+    def card_marking_words_for_length(self, length):
+        # --- Unoptimized version:
+        #num_bits = ((length-1) >> self.card_page_shift) + 1
+        #return (num_bits + (LONG_BIT - 1)) >> LONG_BIT_SHIFT
+        # --- Optimized version:
+        return intmask(
+          ((r_uint(length) + r_uint((LONG_BIT << self.card_page_shift) - 1)) >>
+           (self.card_page_shift + LONG_BIT_SHIFT)))
+
+    def card_marking_bytes_for_length(self, length):
+        # --- Unoptimized version:
+        #num_bits = ((length-1) >> self.card_page_shift) + 1
+        #return (num_bits + 7) >> 3
+        # --- Optimized version:
+        return intmask(
+            ((r_uint(length) + r_uint((8 << self.card_page_shift) - 1)) >>
+             (self.card_page_shift + 3)))
+
+    def debug_check_consistency(self):
+        if self.DEBUG:
+            ll_assert(not self.young_rawmalloced_objects,
+                      "young raw-malloced objects in a major collection")
+            ll_assert(not self.young_objects_with_weakrefs.non_empty(),
+                      "young objects with weakrefs in a major collection")
+
+            if self.raw_malloc_might_sweep.non_empty():
+                ll_assert(self.gc_state == STATE_SWEEPING,
+                      "raw_malloc_might_sweep must be empty outside SWEEPING")
+
+            if self.gc_state == STATE_MARKING:
+                self._debug_objects_to_trace_dict = \
+                                            self.objects_to_trace.stack2dict()
+                MovingGCBase.debug_check_consistency(self)
+                self._debug_objects_to_trace_dict.delete()
+            else:
+                MovingGCBase.debug_check_consistency(self)
+
+    def debug_check_object(self, obj):
+        # We are after a minor collection, and possibly after a major
+        # collection step.  No object should be in the nursery
+        ll_assert(not self.is_in_nursery(obj),
+                  "object in nursery after collection")
+        ll_assert(self.header(obj).tid & GCFLAG_VISITED_RMY == 0,
+                  "GCFLAG_VISITED_RMY after collection")
+
+        if self.gc_state == STATE_SCANNING:
+            self._debug_check_object_scanning(obj)
+        elif self.gc_state == STATE_MARKING:
+            self._debug_check_object_marking(obj)
+        elif self.gc_state == STATE_SWEEPING:
+            self._debug_check_object_sweeping(obj)
+        elif self.gc_state == STATE_FINALIZING:
+            self._debug_check_object_finalizing(obj)
+        else:
+            ll_assert(False, "unknown gc_state value")
+
+    def _debug_check_object_marking(self, obj):
+        if self.header(obj).tid & GCFLAG_VISITED != 0:
+            # A black object.  Should NEVER point to a white object.
+            self.trace(obj, self._debug_check_not_white, None)
+            # During marking, all visited (black) objects should always have
+            # the GCFLAG_TRACK_YOUNG_PTRS flag set, for the write barrier to
+            # trigger --- at least if they contain any gc ptr.  We are just
+            # after a minor or major collection here, so we can't see the
+            # object state VISITED & ~WRITE_BARRIER.
+            typeid = self.get_type_id(obj)
+            if self.has_gcptr(typeid):
+                ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS != 0,
+                          "black object without GCFLAG_TRACK_YOUNG_PTRS")
+
+    def _debug_check_not_white(self, root, ignored):
+        obj = root.address[0]
+        if self.header(obj).tid & GCFLAG_VISITED != 0:
+            pass    # black -> black
+        elif self._debug_objects_to_trace_dict.contains(obj):
+            pass    # black -> gray
+        elif self.header(obj).tid & GCFLAG_NO_HEAP_PTRS != 0:
+            pass    # black -> white-but-prebuilt-so-dont-care
+        else:
+            ll_assert(False, "black -> white pointer found")
+
+    def _debug_check_object_sweeping(self, obj):
+        # We see only reachable objects here.  They all start as VISITED
+        # but this flag is progressively removed in the sweeping phase.
+
+        # All objects should have this flag, except if they
+        # don't have any GC pointer
+        typeid = self.get_type_id(obj)
+        if self.has_gcptr(typeid):
+            ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS != 0,
+                      "missing GCFLAG_TRACK_YOUNG_PTRS")
+        # the GCFLAG_FINALIZATION_ORDERING should not be set between coll.
+        ll_assert(self.header(obj).tid & GCFLAG_FINALIZATION_ORDERING == 0,
+                  "unexpected GCFLAG_FINALIZATION_ORDERING")
+        # the GCFLAG_CARDS_SET should not be set between collections
+        ll_assert(self.header(obj).tid & GCFLAG_CARDS_SET == 0,
+                  "unexpected GCFLAG_CARDS_SET")
+        # if the GCFLAG_HAS_CARDS is set, check that all bits are zero now
+        if self.header(obj).tid & GCFLAG_HAS_CARDS:
+            if self.card_page_indices <= 0:
+                ll_assert(False, "GCFLAG_HAS_CARDS but not using card marking")
+                return
+            typeid = self.get_type_id(obj)
+            ll_assert(self.has_gcptr_in_varsize(typeid),
+                      "GCFLAG_HAS_CARDS but not has_gcptr_in_varsize")
+            ll_assert(self.header(obj).tid & GCFLAG_NO_HEAP_PTRS == 0,
+                      "GCFLAG_HAS_CARDS && GCFLAG_NO_HEAP_PTRS")
+            offset_to_length = self.varsize_offset_to_length(typeid)
+            length = (obj + offset_to_length).signed[0]
+            extra_words = self.card_marking_words_for_length(length)
+            #
+            size_gc_header = self.gcheaderbuilder.size_gc_header
+            p = llarena.getfakearenaaddress(obj - size_gc_header)
+            i = extra_words * WORD
+            while i > 0:
+                p -= 1
+                ll_assert(p.char[0] == '\x00',
+                          "the card marker bits are not cleared")
+                i -= 1
+
+    def _debug_check_object_finalizing(self, obj):
+        # Same invariants as STATE_SCANNING.
+        self._debug_check_object_scanning(obj)
+
+    def _debug_check_object_scanning(self, obj):
+        # This check is called before scanning starts.
+        # Scanning is done in a single step.
+        # the GCFLAG_VISITED should not be set between collections
+        ll_assert(self.header(obj).tid & GCFLAG_VISITED == 0,
+                  "unexpected GCFLAG_VISITED")
+
+        # All other invariants from the sweeping phase should still be
+        # satisfied.
+        self._debug_check_object_sweeping(obj)
+
+
+    # ----------
+    # Write barrier
+
+    # for the JIT: a minimal description of the write_barrier() method
+    # (the JIT assumes it is of the shape
+    #  "if addr_struct.int0 & JIT_WB_IF_FLAG: remember_young_pointer()")
+    JIT_WB_IF_FLAG = GCFLAG_TRACK_YOUNG_PTRS
+
+    # for the JIT to generate custom code corresponding to the array
+    # write barrier for the simplest case of cards.  If JIT_CARDS_SET
+    # is already set on an object, it will execute code like this:
+    #    MOV eax, index
+    #    SHR eax, JIT_WB_CARD_PAGE_SHIFT
+    #    XOR eax, -8
+    #    BTS [object], eax
+    if TRANSLATION_PARAMS['card_page_indices'] > 0:
+        JIT_WB_CARDS_SET = GCFLAG_CARDS_SET
+        JIT_WB_CARD_PAGE_SHIFT = 1
+        while ((1 << JIT_WB_CARD_PAGE_SHIFT) !=
+               TRANSLATION_PARAMS['card_page_indices']):
+            JIT_WB_CARD_PAGE_SHIFT += 1
+
+    @classmethod
+    def JIT_max_size_of_young_obj(cls):
+        return cls.TRANSLATION_PARAMS['large_object']
+
+    @classmethod
+    def JIT_minimal_size_in_nursery(cls):
+        return cls.minimal_size_in_nursery
+
+    def write_barrier(self, addr_struct):
+        if self.header(addr_struct).tid & GCFLAG_TRACK_YOUNG_PTRS:
+            self.remember_young_pointer(addr_struct)
+
+    def write_barrier_from_array(self, addr_array, index):
+        if self.header(addr_array).tid & GCFLAG_TRACK_YOUNG_PTRS:
+            if self.card_page_indices > 0:
+                self.remember_young_pointer_from_array2(addr_array, index)
+            else:
+                self.remember_young_pointer(addr_array)
+
+    def _init_writebarrier_logic(self):
+        DEBUG = self.DEBUG
+        # The purpose of attaching remember_young_pointer to the instance
+        # instead of keeping it as a regular method is to
+        # make the code in write_barrier() marginally smaller
+        # (which is important because it is inlined *everywhere*).
+        def remember_young_pointer(addr_struct):
+            # 'addr_struct' is the address of the object in which we write.
+            # We know that 'addr_struct' has GCFLAG_TRACK_YOUNG_PTRS so far.
+            #
+            if DEBUG:   # note: PYPY_GC_DEBUG=1 does not enable this
+                ll_assert(self.debug_is_old_object(addr_struct) or
+                          self.header(addr_struct).tid & GCFLAG_HAS_CARDS != 0,
+                      "young object with GCFLAG_TRACK_YOUNG_PTRS and no cards")
+            #
+            # We need to remove the flag GCFLAG_TRACK_YOUNG_PTRS and add
+            # the object to the list 'old_objects_pointing_to_young'.
+            # We know that 'addr_struct' cannot be in the nursery,
+            # because nursery objects never have the flag
+            # GCFLAG_TRACK_YOUNG_PTRS to start with.  Note that in
+            # theory we don't need to do that if the pointer that we're
+            # writing into the object isn't pointing to a young object.
+            # However, it isn't really a win, because then sometimes
+            # we're going to call this function a lot of times for the
+            # same object; moreover we'd need to pass the 'newvalue' as
+            # an argument here.  The JIT has always called a
+            # 'newvalue'-less version, too.  Moreover, the incremental
+            # GC nowadays relies on this fact.
+            self.old_objects_pointing_to_young.append(addr_struct)
+            objhdr = self.header(addr_struct)
+            objhdr.tid &= ~GCFLAG_TRACK_YOUNG_PTRS
+            #
+            # Second part: if 'addr_struct' is actually a prebuilt GC
+            # object and it's the first time we see a write to it, we
+            # add it to the list 'prebuilt_root_objects'.
+            if objhdr.tid & GCFLAG_NO_HEAP_PTRS:
+                objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS
+                self.prebuilt_root_objects.append(addr_struct)
+
+        remember_young_pointer._dont_inline_ = True
+        self.remember_young_pointer = remember_young_pointer
+        #
+        if self.card_page_indices > 0:
+            self._init_writebarrier_with_card_marker()
+
+
+    def _init_writebarrier_with_card_marker(self):
+        DEBUG = self.DEBUG
+        def remember_young_pointer_from_array2(addr_array, index):
+            # 'addr_array' is the address of the object in which we write,
+            # which must have an array part;  'index' is the index of the
+            # item that is (or contains) the pointer that we write.
+            # We know that 'addr_array' has GCFLAG_TRACK_YOUNG_PTRS so far.
+            #
+            objhdr = self.header(addr_array)
+            if objhdr.tid & GCFLAG_HAS_CARDS == 0:
+                #
+                if DEBUG:   # note: PYPY_GC_DEBUG=1 does not enable this
+                    ll_assert(self.debug_is_old_object(addr_array),
+                        "young array with no card but GCFLAG_TRACK_YOUNG_PTRS")
+                #
+                # no cards, use default logic.  Mostly copied from above.
+                self.old_objects_pointing_to_young.append(addr_array)
+                objhdr.tid &= ~GCFLAG_TRACK_YOUNG_PTRS
+                if objhdr.tid & GCFLAG_NO_HEAP_PTRS:
+                    objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS
+                    self.prebuilt_root_objects.append(addr_array)
+                return
+            #
+            # 'addr_array' is a raw_malloc'ed array with card markers
+            # in front.  Compute the index of the bit to set:
+            bitindex = index >> self.card_page_shift
+            byteindex = bitindex >> 3
+            bitmask = 1 << (bitindex & 7)
+            #
+            # If the bit is already set, leave now.
+            addr_byte = self.get_card(addr_array, byteindex)
+            byte = ord(addr_byte.char[0])
+            if byte & bitmask:
+                return
+            #
+            # We set the flag (even if the newly written address does not
+            # actually point to the nursery, which seems to be ok -- actually
+            # it seems more important that remember_young_pointer_from_array2()
+            # does not take 3 arguments).
+            addr_byte.char[0] = chr(byte | bitmask)
+            #
+            if objhdr.tid & GCFLAG_CARDS_SET == 0:
+                self.old_objects_with_cards_set.append(addr_array)
+                objhdr.tid |= GCFLAG_CARDS_SET
+
+        remember_young_pointer_from_array2._dont_inline_ = True
+        assert self.card_page_indices > 0
+        self.remember_young_pointer_from_array2 = (
+            remember_young_pointer_from_array2)
+
+        def jit_remember_young_pointer_from_array(addr_array):
+            # minimal version of the above, with just one argument,
+            # called by the JIT when GCFLAG_TRACK_YOUNG_PTRS is set
+            # but GCFLAG_CARDS_SET is cleared.  This tries to set
+            # GCFLAG_CARDS_SET if possible; otherwise, it falls back
+            # to remember_young_pointer().
+            objhdr = self.header(addr_array)
+            if objhdr.tid & GCFLAG_HAS_CARDS:
+                self.old_objects_with_cards_set.append(addr_array)
+                objhdr.tid |= GCFLAG_CARDS_SET
+            else:
+                self.remember_young_pointer(addr_array)
+
+        self.jit_remember_young_pointer_from_array = (
+            jit_remember_young_pointer_from_array)
+
+    def get_card(self, obj, byteindex):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        addr_byte = obj - size_gc_header
+        return llarena.getfakearenaaddress(addr_byte) + (~byteindex)
+
+
+    def writebarrier_before_copy(self, source_addr, dest_addr,
+                                 source_start, dest_start, length):
+        """ This has the same effect as calling writebarrier over
+        each element in dest copied from source, except it might reset
+        one of the following flags a bit too eagerly, which means we'll have
+        a bit more objects to track, but being on the safe side.
+        """
+        source_hdr = self.header(source_addr)
+        dest_hdr = self.header(dest_addr)
+        if dest_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
+            return True
+        # ^^^ a fast path of write-barrier
+        #
+        if source_hdr.tid & GCFLAG_HAS_CARDS != 0:
+            #
+            if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
+                # The source object may have random young pointers.
+                # Return False to mean "do it manually in ll_arraycopy".
+                return False
+            #
+            if source_hdr.tid & GCFLAG_CARDS_SET == 0:
+                # The source object has no young pointers at all.  Done.
+                return True
+            #
+            if dest_hdr.tid & GCFLAG_HAS_CARDS == 0:
+                # The dest object doesn't have cards.  Do it manually.
+                return False
+            #
+            if source_start != 0 or dest_start != 0:
+                # Misaligned.  Do it manually.
+                return False
+            #
+            self.manually_copy_card_bits(source_addr, dest_addr, length)
+            return True
+        #
+        if source_hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
+            # there might be in source a pointer to a young object
+            self.old_objects_pointing_to_young.append(dest_addr)
+            dest_hdr.tid &= ~GCFLAG_TRACK_YOUNG_PTRS
+        #
+        if dest_hdr.tid & GCFLAG_NO_HEAP_PTRS:
+            if source_hdr.tid & GCFLAG_NO_HEAP_PTRS == 0:
+                dest_hdr.tid &= ~GCFLAG_NO_HEAP_PTRS
+                self.prebuilt_root_objects.append(dest_addr)
+        return True
+
+    def manually_copy_card_bits(self, source_addr, dest_addr, length):
+        # manually copy the individual card marks from source to dest
+        assert self.card_page_indices > 0
+        bytes = self.card_marking_bytes_for_length(length)
+        #
+        anybyte = 0
+        i = 0
+        while i < bytes:
+            addr_srcbyte = self.get_card(source_addr, i)
+            addr_dstbyte = self.get_card(dest_addr, i)
+            byte = ord(addr_srcbyte.char[0])
+            anybyte |= byte
+            addr_dstbyte.char[0] = chr(ord(addr_dstbyte.char[0]) | byte)
+            i += 1
+        #
+        if anybyte:
+            dest_hdr = self.header(dest_addr)
+            if dest_hdr.tid & GCFLAG_CARDS_SET == 0:
+                self.old_objects_with_cards_set.append(dest_addr)
+                dest_hdr.tid |= GCFLAG_CARDS_SET
+
+    # ----------
+    # Nursery collection
+
+    def minor_collection(self):
+        """Perform a minor collection: find the objects from the nursery
+        that remain alive and move them out."""
+        #
+        debug_start("gc-minor")
+        #
+        # Before everything else, remove from 'old_objects_pointing_to_young'
+        # the young arrays.
+        if self.young_rawmalloced_objects:
+            self.remove_young_arrays_from_old_objects_pointing_to_young()
+        #
+        # First, find the roots that point to young objects.  All nursery
+        # objects found are copied out of the nursery, and the occasional
+        # young raw-malloced object is flagged with GCFLAG_VISITED_RMY.
+        # Note that during this step, we ignore references to further
+        # young objects; only objects directly referenced by roots
+        # are copied out or flagged.  They are also added to the list
+        # 'old_objects_pointing_to_young'.
+        self.nursery_surviving_size = 0
+        self.collect_roots_in_nursery()
+        #
+        while True:
+            # If we are using card marking, do a partial trace of the arrays
+            # that are flagged with GCFLAG_CARDS_SET.
+            if self.card_page_indices > 0:
+                self.collect_cardrefs_to_nursery()
+            #
+            # Now trace objects from 'old_objects_pointing_to_young'.
+            # All nursery objects they reference are copied out of the
+            # nursery, and again added to 'old_objects_pointing_to_young'.
+            # All young raw-malloced object found are flagged
+            # GCFLAG_VISITED_RMY.
+            # We proceed until 'old_objects_pointing_to_young' is empty.
+            self.collect_oldrefs_to_nursery()
+            #
+            # We have to loop back if collect_oldrefs_to_nursery caused
+            # new objects to show up in old_objects_with_cards_set
+            if self.card_page_indices > 0:
+                if self.old_objects_with_cards_set.non_empty():
+                    continue
+            break
+        #
+        # Now all live nursery objects should be out.  Update the young
+        # weakrefs' targets.
+        if self.young_objects_with_weakrefs.non_empty():
+            self.invalidate_young_weakrefs()
+        if self.young_objects_with_light_finalizers.non_empty():
+            self.deal_with_young_objects_with_finalizers()
+        #
+        # Clear this mapping.
+        if self.nursery_objects_shadows.length() > 0:
+            self.nursery_objects_shadows.clear()
+        #
+        # Walk the list of young raw-malloced objects, and either free
+        # them or make them old.
+        if self.young_rawmalloced_objects:
+            self.free_young_rawmalloced_objects()
+        #
+        # All live nursery objects are out, and the rest dies.  Fill
+        # the nursery up to the cleanup point with zeros
+        llarena.arena_reset(self.nursery, self.nursery_size, 0)
+        llarena.arena_reset(self.nursery, self.initial_cleanup, 2)
+        self.debug_rotate_nursery()
+        self.nursery_free = self.nursery
+        self.nursery_top = self.nursery + self.initial_cleanup
+        self.nursery_real_top = self.nursery + self.nursery_size
+        #
+        debug_print("minor collect, total memory used:",
+                    self.get_total_memory_used())
+        if self.DEBUG >= 2:
+            self.debug_check_consistency()     # expensive!
+        debug_stop("gc-minor")
+
+
+    def collect_roots_in_nursery(self):
+        # we don't need to trace prebuilt GcStructs during a minor collect:
+        # if a prebuilt GcStruct contains a pointer to a young object,
+        # then the write_barrier must have ensured that the prebuilt
+        # GcStruct is in the list self.old_objects_pointing_to_young.
+        debug_start("gc-minor-walkroots")
+        self.root_walker.walk_roots(
+            IncrementalMiniMarkGC._trace_drag_out1,  # stack roots
+            IncrementalMiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
+            None)                         # static in prebuilt gc
+        debug_stop("gc-minor-walkroots")
+
+    def collect_cardrefs_to_nursery(self):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        oldlist = self.old_objects_with_cards_set
+        while oldlist.non_empty():
+            obj = oldlist.pop()
+            #
+            # Remove the GCFLAG_CARDS_SET flag.
+            ll_assert(self.header(obj).tid & GCFLAG_CARDS_SET != 0,
+                "!GCFLAG_CARDS_SET but object in 'old_objects_with_cards_set'")
+            self.header(obj).tid &= ~GCFLAG_CARDS_SET
+            #
+            # Get the number of card marker bytes in the header.
+            typeid = self.get_type_id(obj)
+            offset_to_length = self.varsize_offset_to_length(typeid)
+            length = (obj + offset_to_length).signed[0]
+            bytes = self.card_marking_bytes_for_length(length)
+            p = llarena.getfakearenaaddress(obj - size_gc_header)
+            #
+            # If the object doesn't have GCFLAG_TRACK_YOUNG_PTRS, then it
+            # means that it is in 'old_objects_pointing_to_young' and
+            # will be fully traced by collect_oldrefs_to_nursery() just
+            # afterwards.
+            if self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
+                #
+                # In that case, we just have to reset all card bits.
+                while bytes > 0:
+                    p -= 1
+                    p.char[0] = '\x00'
+                    bytes -= 1
+                #
+            else:
+                # Walk the bytes encoding the card marker bits, and for
+                # each bit set, call trace_and_drag_out_of_nursery_partial().
+                interval_start = 0
+                while bytes > 0:
+                    p -= 1
+                    cardbyte = ord(p.char[0])
+                    p.char[0] = '\x00'           # reset the bits
+                    bytes -= 1
+                    next_byte_start = interval_start + 8*self.card_page_indices
+                    #
+                    while cardbyte != 0:
+                        interval_stop = interval_start + self.card_page_indices
+                        #
+                        if cardbyte & 1:
+                            if interval_stop > length:
+                                interval_stop = length
+                                ll_assert(cardbyte <= 1 and bytes == 0,
+                                          "premature end of object")
+                            self.trace_and_drag_out_of_nursery_partial(
+                                obj, interval_start, interval_stop)
+                        #
+                        interval_start = interval_stop
+                        cardbyte >>= 1
+                    interval_start = next_byte_start
+                #
+                # If we're incrementally marking right now, sorry, we also
+                # need to add the object to 'objects_to_trace' and have it
+                # fully traced very soon.
+                if self.gc_state == STATE_MARKING:
+                    self.header(obj).tid &= ~GCFLAG_VISITED
+                    self.objects_to_trace.append(obj)
+
+
+    def collect_oldrefs_to_nursery(self):
+        if self.gc_state == STATE_MARKING:
+            self._collect_oldrefs_to_nursery(True)
+        else:
+            self._collect_oldrefs_to_nursery(False)
+
+    @specialize.arg(1)
+    def _collect_oldrefs_to_nursery(self, state_is_marking):
+        # Follow the old_objects_pointing_to_young list and move the
+        # young objects they point to out of the nursery.
+        oldlist = self.old_objects_pointing_to_young
+        while oldlist.non_empty():
+            obj = oldlist.pop()
+            #
+            # Check that the flags are correct: we must not have
+            # GCFLAG_TRACK_YOUNG_PTRS so far.
+            ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS == 0,
+                      "old_objects_pointing_to_young contains obj with "
+                      "GCFLAG_TRACK_YOUNG_PTRS")
+            #
+            # Add the flag GCFLAG_TRACK_YOUNG_PTRS.  All live objects should
+            # have this flag set after a nursery collection.
+            self.header(obj).tid |= GCFLAG_TRACK_YOUNG_PTRS
+            #
+            # If the incremental major collection is currently at
+            # STATE_MARKING, then we must add to 'objects_to_trace' all
+            # objects that go through 'old_objects_pointing_to_young'.
+            # This basically turns black objects gray again, but also
+            # makes sure that we see otherwise-white objects.
+            if state_is_marking:
+                self.header(obj).tid &= ~GCFLAG_VISITED
+                self.objects_to_trace.append(obj)
+            #
+            # Trace the 'obj' to replace pointers to nursery with pointers
+            # outside the nursery, possibly forcing nursery objects out
+            # and adding them to 'old_objects_pointing_to_young' as well.
+            self.trace_and_drag_out_of_nursery(obj)
+
+    def trace_and_drag_out_of_nursery(self, obj):
+        """obj must not be in the nursery.  This copies all the
+        young objects it references out of the nursery.
+        """
+        self.trace(obj, self._trace_drag_out, None)
+
+    def trace_and_drag_out_of_nursery_partial(self, obj, start, stop):
+        """Like trace_and_drag_out_of_nursery(), but limited to the array
+        indices in range(start, stop).
+        """
+        ll_assert(start < stop, "empty or negative range "
+                                "in trace_and_drag_out_of_nursery_partial()")
+        #print 'trace_partial:', start, stop, '\t', obj
+        self.trace_partial(obj, start, stop, self._trace_drag_out, None)
+
+
+    def _trace_drag_out1(self, root):
+        self._trace_drag_out(root, None)
+
+    def _trace_drag_out(self, root, ignored):
+        obj = root.address[0]
+        #print '_trace_drag_out(%x: %r)' % (hash(obj.ptr._obj), obj)
+        #
+        # If 'obj' is not in the nursery, nothing to change -- expect
+        # that we must set GCFLAG_VISITED_RMY on young raw-malloced objects.
+        if not self.is_in_nursery(obj):
+            # cache usage trade-off: I think that it is a better idea to
+            # check if 'obj' is in young_rawmalloced_objects with an access
+            # to this (small) dictionary, rather than risk a lot of cache
+            # misses by reading a flag in the header of all the 'objs' that
+            # arrive here.
+            if (bool(self.young_rawmalloced_objects)
+                and self.young_rawmalloced_objects.contains(obj)):
+                self._visit_young_rawmalloced_object(obj)
+            return
+        #
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
+            #
+            # Common case: 'obj' was not already forwarded (otherwise
+            # tid == -42, containing all flags), and it doesn't have the
+            # HAS_SHADOW flag either.  We must move it out of the nursery,
+            # into a new nonmovable location.
+            totalsize = size_gc_header + self.get_size(obj)
+            self.nursery_surviving_size += raw_malloc_usage(totalsize)
+            newhdr = self._malloc_out_of_nursery(totalsize)
+            #
+        elif self.is_forwarded(obj):
+            #
+            # 'obj' was already forwarded.  Change the original reference
+            # to point to its forwarding address, and we're done.
+            root.address[0] = self.get_forwarding_address(obj)
+            return
+            #
+        else:
+            # First visit to an object that has already a shadow.
+            newobj = self.nursery_objects_shadows.get(obj)
+            ll_assert(newobj != NULL, "GCFLAG_HAS_SHADOW but no shadow found")
+            newhdr = newobj - size_gc_header
+            #
+            # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
+            # copied to the shadow itself.
+            self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+            #
+            totalsize = size_gc_header + self.get_size(obj)
+        #
+        # Copy it.  Note that references to other objects in the
+        # nursery are kept unchanged in this step.
+        llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
+        #
+        # Set the old object's tid to -42 (containing all flags) and
+        # replace the old object's content with the target address.
+        # A bit of no-ops to convince llarena that we are changing
+        # the layout, in non-translated versions.
+        typeid = self.get_type_id(obj)
+        obj = llarena.getfakearenaaddress(obj)
+        llarena.arena_reset(obj - size_gc_header, totalsize, 0)
+        llarena.arena_reserve(obj - size_gc_header,
+                              size_gc_header + llmemory.sizeof(FORWARDSTUB))
+        self.header(obj).tid = -42
+        newobj = newhdr + size_gc_header
+        llmemory.cast_adr_to_ptr(obj, FORWARDSTUBPTR).forw = newobj
+        #
+        # Change the original pointer to this object.
+        root.address[0] = newobj
+        #
+        # Add the newobj to the list 'old_objects_pointing_to_young',
+        # because it can contain further pointers to other young objects.
+        # We will fix such references to point to the copy of the young
+        # objects when we walk 'old_objects_pointing_to_young'.
+        if self.has_gcptr(typeid):
+            # we only have to do it if we have any gcptrs
+            self.old_objects_pointing_to_young.append(newobj)
+        else:
+            # we don't need to add this to 'old_objects_pointing_to_young',
+            # but in the STATE_MARKING phase we still need this bit...
+            if self.gc_state == STATE_MARKING:
+                self.header(newobj).tid &= ~GCFLAG_VISITED
+                self.objects_to_trace.append(newobj)
+
+    _trace_drag_out._always_inline_ = True
+
+    def _visit_young_rawmalloced_object(self, obj):
+        # 'obj' points to a young, raw-malloced object.
+        # Any young rawmalloced object never seen by the code here
+        # will end up without GCFLAG_VISITED_RMY, and be freed at the
+        # end of the current minor collection.  Note that there was
+        # a bug in which dying young arrays with card marks would
+        # still be scanned before being freed, keeping a lot of
+        # objects unnecessarily alive.
+        hdr = self.header(obj)
+        if hdr.tid & GCFLAG_VISITED_RMY:
+            return
+        hdr.tid |= GCFLAG_VISITED_RMY
+        #
+        # we just made 'obj' old, so we need to add it to the correct lists
+        added_somewhere = False
+        #
+        if hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
+            self.old_objects_pointing_to_young.append(obj)
+            added_somewhere = True
+        #
+        if hdr.tid & GCFLAG_HAS_CARDS != 0:
+            ll_assert(hdr.tid & GCFLAG_CARDS_SET != 0,
+                      "young array: GCFLAG_HAS_CARDS without GCFLAG_CARDS_SET")
+            self.old_objects_with_cards_set.append(obj)
+            added_somewhere = True
+        #
+        ll_assert(added_somewhere, "wrong flag combination on young array")
+
+
+    def _malloc_out_of_nursery(self, totalsize):
+        """Allocate non-movable memory for an object of the given
+        'totalsize' that lives so far in the nursery."""
+        if raw_malloc_usage(totalsize) <= self.small_request_threshold:
+            # most common path
+            return self.ac.malloc(totalsize)
+        else:
+            # for nursery objects that are not small
+            return self._malloc_out_of_nursery_nonsmall(totalsize)
+    _malloc_out_of_nursery._always_inline_ = True
+
+    def _malloc_out_of_nursery_nonsmall(self, totalsize):
+        # 'totalsize' should be aligned.
+        ll_assert(raw_malloc_usage(totalsize) & (WORD-1) == 0,
+                  "misaligned totalsize in _malloc_out_of_nursery_nonsmall")
+        #
+        arena = llarena.arena_malloc(raw_malloc_usage(totalsize), False)
+        if not arena:
+            raise MemoryError("cannot allocate object")
+        llarena.arena_reserve(arena, totalsize)
+        #
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        self.rawmalloced_total_size += r_uint(raw_malloc_usage(totalsize))
+        self.old_rawmalloced_objects.append(arena + size_gc_header)
+        return arena
+
+    def free_young_rawmalloced_objects(self):
+        self.young_rawmalloced_objects.foreach(
+            self._free_young_rawmalloced_obj, None)
+        self.young_rawmalloced_objects.delete()
+        self.young_rawmalloced_objects = self.null_address_dict()
+
+    def _free_young_rawmalloced_obj(self, obj, ignored1, ignored2):
+        # If 'obj' has GCFLAG_VISITED_RMY, it was seen by _trace_drag_out
+        # and survives.  Otherwise, it dies.
+        self.free_rawmalloced_object_if_unvisited(obj, GCFLAG_VISITED_RMY)
+
+    def remove_young_arrays_from_old_objects_pointing_to_young(self):
+        old = self.old_objects_pointing_to_young
+        new = self.AddressStack()
+        while old.non_empty():
+            obj = old.pop()
+            if not self.young_rawmalloced_objects.contains(obj):
+                new.append(obj)
+        # an extra copy, to avoid assignments to
+        # 'self.old_objects_pointing_to_young'
+        while new.non_empty():
+            old.append(new.pop())
+        new.delete()
+
+    def minor_and_major_collection(self):
+        # First, finish the current major gc, if there is one in progress.
+        # This is a no-op if the gc_state is already STATE_SCANNING.
+        self.gc_step_until(STATE_SCANNING)
+        #
+        # Then do a complete collection again.
+        self.gc_step_until(STATE_MARKING)
+        self.gc_step_until(STATE_SCANNING)
+
+    def gc_step_until(self, state, reserving_size=0):
+        while self.gc_state != state:
+            self.minor_collection()
+            self.major_collection_step(reserving_size)
+
+    debug_gc_step_until = gc_step_until   # xxx
+
+    def debug_gc_step(self, n=1):
+        while n > 0:
+            self.minor_collection()
+            self.major_collection_step()
+            n -= 1
+
+    # Note - minor collections seem fast enough so that one
+    # is done before every major collection step
+    def major_collection_step(self, reserving_size=0):
+        debug_start("gc-collect-step")
+        debug_print("starting gc state: ", GC_STATES[self.gc_state])
+        # Debugging checks
+        ll_assert(self.nursery_free == self.nursery,
+                  "nursery not empty in major_collection_step()")
+        self.debug_check_consistency()
+
+
+        # XXX currently very course increments, get this working then split
+        # to smaller increments using stacks for resuming
+        if self.gc_state == STATE_SCANNING:
+            self.objects_to_trace = self.AddressStack()
+            self.collect_roots()
+            self.gc_state = STATE_MARKING
+            #END SCANNING
+        elif self.gc_state == STATE_MARKING:
+            debug_print("number of objects to mark",
+                        self.objects_to_trace.length())
+            estimate = self.gc_increment_step
+            estimate_from_nursery = self.nursery_surviving_size * 2
+            if estimate_from_nursery > estimate:
+                estimate = estimate_from_nursery
+            self.visit_all_objects_step(intmask(estimate))
+
+            # XXX A simplifying assumption that should be checked,
+            # finalizers/weak references are rare and short which means that
+            # they do not need a seperate state and do not need to be
+            # made incremental.
+            if not self.objects_to_trace.non_empty():
+                if self.objects_with_finalizers.non_empty():
+                    self.deal_with_objects_with_finalizers()
+
+                self.objects_to_trace.delete()
+
+                #
+                # Weakref support: clear the weak pointers to dying objects
+                if self.old_objects_with_weakrefs.non_empty():
+                    self.invalidate_old_weakrefs()
+                if self.old_objects_with_light_finalizers.non_empty():
+                    self.deal_with_old_objects_with_finalizers()
+                #objects_to_trace processed fully, can move on to sweeping
+                self.ac.mass_free_prepare()
+                self.start_free_rawmalloc_objects()
+                self.gc_state = STATE_SWEEPING
+            #END MARKING
+        elif self.gc_state == STATE_SWEEPING:
+            #
+            # Walk all rawmalloced objects and free the ones that don't
+            # have the GCFLAG_VISITED flag.  Visit at most 'limit' objects.
+            limit = self.nursery_size // self.ac.page_size
+            remaining = self.free_unvisited_rawmalloc_objects_step(limit)
+            #
+            # Ask the ArenaCollection to visit a fraction of the objects.
+            # Free the ones that have not been visited above, and reset
+            # GCFLAG_VISITED on the others.  Visit at most '3 * limit'
+            # pages minus the number of objects already visited above.
+            done = self.ac.mass_free_incremental(self._free_if_unvisited,
+                                                 2 * limit + remaining)
+            # XXX tweak the limits above
+            #
+            if remaining > 0 and done:
+                self.num_major_collects += 1
+                #
+                # We also need to reset the GCFLAG_VISITED on prebuilt GC objects.
+                self.prebuilt_root_objects.foreach(self._reset_gcflag_visited, None)
+                #
+                # Set the threshold for the next major collection to be when we
+                # have allocated 'major_collection_threshold' times more than
+                # we currently have -- but no more than 'max_delta' more than
+                # we currently have.
+                total_memory_used = float(self.get_total_memory_used())
+                bounded = self.set_major_threshold_from(
+                    min(total_memory_used * self.major_collection_threshold,
+                        total_memory_used + self.max_delta),
+                    reserving_size)
+                #
+                # Max heap size: gives an upper bound on the threshold.  If we
+                # already have at least this much allocated, raise MemoryError.
+                if bounded and (float(self.get_total_memory_used()) + reserving_size >=
+                                self.next_major_collection_initial):
+                    #
+                    # First raise MemoryError, giving the program a chance to
+                    # quit cleanly.  It might still allocate in the nursery,
+                    # which might eventually be emptied, triggering another
+                    # major collect and (possibly) reaching here again with an
+                    # even higher memory consumption.  To prevent it, if it's
+                    # the second time we are here, then abort the program.
+                    if self.max_heap_size_already_raised:
+                        llop.debug_fatalerror(lltype.Void,
+                                              "Using too much memory, aborting")
+                    self.max_heap_size_already_raised = True
+                    raise MemoryError
+
+                self.gc_state = STATE_FINALIZING
+            # FINALIZING not yet incrementalised
+            # but it seems safe to allow mutator to run after sweeping and
+            # before finalizers are called. This is because run_finalizers
+            # is a different list to objects_with_finalizers.
+            # END SWEEPING
+        elif self.gc_state == STATE_FINALIZING:
+            # XXX This is considered rare,
+            # so should we make the calling incremental? or leave as is
+
+            # Must be ready to start another scan

From noreply at buildbot.pypy.org  Mon Oct 14 18:23:09 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 14 Oct 2013 18:23:09 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Some grammar and language tweaks
Message-ID: <20131014162309.C5B4F1C02C2@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5075:cfedeaf7332e
Date: 2013-10-14 18:21 +0200
http://bitbucket.org/pypy/extradoc/changeset/cfedeaf7332e/

Log:	Some grammar and language tweaks

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -1,10 +1,9 @@
-
 Incremental Garbage Collector in PyPy
 =====================================
 
 Hello everyone.
 
-We're pleased to announce that as of today (so tomorrows nightly),
+We're pleased to announce that as of today (so tomorrow's nightly),
 the default PyPy comes with a GC that has much smaller pauses than yesterday.
 
 Let's start with explaining roughly what GC pauses are. In CPython each
@@ -23,19 +22,19 @@
    del a
    del b
 
-This creates a reference cycles. It means that while we deleted references to
-``a`` and ``b`` from the current scope, they still have a refcount of 1,
-because they point to each other, while the whole group has no reference
-from the outside. CPython employs a cyclic garbage collector that is used to
-find such cycles. It walks all objects in memory, starting from known roots
-like immortal prebuilt objects, roots on the stack etc. This solves the
-problem, but creates a noticable GC pauses when the heap becomes large and
+This creates a reference cycle. It means that while we deleted references to
+``a`` and ``b`` from the current scope, they still have a reference count of 1,
+because they point to each other, even though the whole group has no references
+from the outside. CPython employs a cyclic garbage collector which is used to
+find such cycles. It walks over all objects in memory, starting from some known
+roots, such as ``type`` objects, variables on the stack, etc. This solves the
+problem, but can create noticable GC pauses as the heap becomes large and
 convoluted.
 
-PyPy has essentially only the cycle finder - it does not bother with reference
-counting, but it walks alive objects every now and then (this is a big
-simplification, PyPy's GC is much more complex than this). It also has
-a problem of GC pauses. To alleviate this problem, which is essential for
+PyPy essentially has only the cycle finder - it does not bother with reference
+counting, instead it walks alive objects every now and then (this is a big
+simplification, PyPy's GC is much more complex than this). As a result it also
+has the problem of GC pauses. To alleviate this problem, which is essential for
 applications like games, we started to work on incremental GC, which spreads
 the walking of objects and cleaning them across the execution time in smaller
 intervals. The work was sponsored by the Raspberry Pi foundation, started

From noreply at buildbot.pypy.org  Mon Oct 14 18:24:20 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 14 Oct 2013 18:24:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: Kill the TODO file from
 'incremental-gc'. Most things have been fixed
Message-ID: <20131014162420.16C471C02C2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67366:f5a80e460c19
Date: 2013-10-14 18:23 +0200
http://bitbucket.org/pypy/pypy/changeset/f5a80e460c19/

Log:	Kill the TODO file from 'incremental-gc'. Most things have been
	fixed already.

diff --git a/TODO b/TODO
deleted file mode 100644
--- a/TODO
+++ /dev/null
@@ -1,24 +0,0 @@
-
-
-* fix increments in major_collection_step() in the phases
-  STATE_MARKING, STATE_SWEEPING_RAWMALLOC, STATE_SWEEPING_ARENA,
-  and probably STATE_FINALIZING
-
-* 'next_major_collection_*' need to be tweaked
-
-* check the 'reserving_size' argument here and there
-
-* maybe make incremental: dealing with old_objects_with_weakrefs
-  and old_objects_with_light_finalizers and
-  deal_with_objects_with_finalizers()
-
-* REDO external_malloc(): if somebody calls this function a lot, we must
-  eventually force a full collection.
-
-* REDO card marking, starting with "card_page_indices": 128 in
-  TRANSLATION_PARAMS
-
-* write barrier: avoid the case when during sweeping we have GCFLAG_VISITED
-  on an object, so we call the slow path, but the slow path doesn't do
-  anything, and we still have GCFLAG_VISITED so we will keep calling it
-  on the same object

From noreply at buildbot.pypy.org  Mon Oct 14 18:24:59 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 14 Oct 2013 18:24:59 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Add an XXX
Message-ID: <20131014162459.0899D1D22C2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5076:8bd16d22bf53
Date: 2013-10-14 18:24 +0200
http://bitbucket.org/pypy/extradoc/changeset/8bd16d22bf53/

Log:	Add an XXX

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -9,6 +9,7 @@
 Let's start with explaining roughly what GC pauses are. In CPython each
 object has a reference count, which is incremented each time we create
 references and decremented each time we forget them. This means that objects
+(XXX also, very long chains of objects cause unbounded pauses in CPython)
 are freed each time they become unreachable. That is only half of the story
 though. Consider code like this::
 

From noreply at buildbot.pypy.org  Mon Oct 14 20:08:21 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 14 Oct 2013 20:08:21 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Nitty gritty details: starting.
	May become too long
Message-ID: <20131014180821.3C5061C067F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5077:e25ed3d1866c
Date: 2013-10-14 20:08 +0200
http://bitbucket.org/pypy/extradoc/changeset/e25ed3d1866c/

Log:	Nitty gritty details: starting. May become too long

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -49,4 +49,69 @@
 Nitty gritty details
 ====================
 
+This was done as a patch to "minimark", our current GC, and called
+"incminimark" for now.  The former is a generational stop-the-world GC.
+New objects are allocated "young", i.e. in the nursery, a special zone
+of a few MB of memory.  When it is full, a "minor collection" step moves
+the surviving objects out of the nursery.  This can be done quickly (a
+few millisecond at most) because we only need to walk through the young
+objects that survive --- usually a small fraction of all young objects.
+From time to time, this minor collection is followed by a "major
+collection": in that step, we walk *all* objects to classify which ones
+are still alive and which ones are now dead (*marking*) and free the
+memory occupied by the dead ones (*speeding*).
 
+This "major collection" is what gives the long GC pauses.  To fix this
+problem we made the GC incremental: instead of running one complete
+major collection, we split its work into a variable number of pieces
+and run each piece after every minor collection for a while, until there
+are no more pieces.  The pieces are each doing a fraction of marking, or
+a fraction of sweeping.
+
+The main issue is that splitting the major collections means that the
+main program is actually running between the pieces, and so can change
+the pointers in the objects to point to other objects.  This is not
+a problem for sweeping: dead objects will remain dead whatever the main
+program does.  However, it is a problem for marking.  Let us see why.
+
+In terms of the incremental GC literature, objects are either "white",
+"gray" or "black".  They start as "white", become "gray" when they are
+found to be alive, and become "black" when they have been fully
+traversed --- at which point the objects that it points to have
+themselves been marked gray, or maybe are already black.  The gray
+objects are the "frontier" between the black objects that we have found
+to be reachable, and the white objects that represent the unknown part
+of the world.  When there are no more gray objects, the process is
+finished: all remaining white objects are unreachable and can be freed
+(by the following sweeping phase).
+
+In this model, the important part is that a black object can never point
+to a white object: if the latter remains white until the end, it will be
+freed, which is incorrect because the black object itself can still be
+reached.
+
+The trick we used in PyPy is to consider minor collections as part of
+the whole, rather than focus only on major collections.  The existing
+minimark GC had always used a "write barrier" to do its job, like any
+generational GC.  This write barrier is used to detect when an old
+object (outside the nursery) is modified to point to a young object
+(inside the nursery), which is essential information for minor
+collections.  Actually, although this was the goal, the actual write
+barrier code was simpler: it just recorded all old objects into which we
+wrote *any* pointer --- to a young or old object.  It is actually a
+performance improvement, because we don't need to check over and over
+again if the written pointer points to a young object or not.
+
+This *unmodified* write barrier works for incminimark too.  Imagine that
+we are in the middle of the marking phase, running the main program.
+The write barrier will record all old objects that are being modified.
+Then at the next minor collection, all surviving young objects will be
+moved out of the nursery.  At this point, as we're about to continue
+running the major collection's marking phase, we simply add to the list
+of pending gray objects all the objects that we consider --- both the
+objects listed as "old objects that are being modified", and the objects
+that we just moved out of the nursery.  A fraction of the former list
+are turned back from the black to the gray color.  This technique
+implements nicely, if indirectly, what is called a "backward write
+barrier" in the literature: the backwardness is about the color that
+occasionally progresses backward from black to gray.

From noreply at buildbot.pypy.org  Mon Oct 14 21:36:49 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 14 Oct 2013 21:36:49 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: A little bit more text
Message-ID: <20131014193649.813AF1C01F7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5078:6ae5b6468121
Date: 2013-10-14 21:36 +0200
http://bitbucket.org/pypy/extradoc/changeset/6ae5b6468121/

Log:	A little bit more text

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -92,7 +92,8 @@
 
 The trick we used in PyPy is to consider minor collections as part of
 the whole, rather than focus only on major collections.  The existing
-minimark GC had always used a "write barrier" to do its job, like any
+minimark GC had always used a "write barrier" (a piece of code run every time
+you set or get from an object or array) to do its job, like any
 generational GC.  This write barrier is used to detect when an old
 object (outside the nursery) is modified to point to a young object
 (inside the nursery), which is essential information for minor

From noreply at buildbot.pypy.org  Mon Oct 14 22:00:52 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 14 Oct 2013 22:00:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: add tests
Message-ID: <20131014200052.77B381C02E2@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67367:ed63cc82bf1b
Date: 2013-10-14 07:03 +0300
http://bitbucket.org/pypy/pypy/changeset/ed63cc82bf1b/

Log:	add tests

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -976,3 +976,21 @@
         assert zeros_like(2) == array(0)
         assert zeros_like(2.) == array(0.)
         assert zeros_like(complex(2)) == array(complex(0))
+
+    def test_accumulate(self):
+        from numpypy import add, multiply, arange
+        assert (add.accumulate([2, 3, 5]) == [2, 5, 10]).all()
+        assert (multiply.accumulate([2, 3, 5]) == [2, 6, 30]).all()
+        a = arange(4).reshape(2,2)
+        b = add.accumulate(a, axis=0)
+        assert (b == [[0, 1], [2, 4]]).all()
+        b = add.accumulate(a, 1)
+        assert (b == [[0, 1], [2, 5]]).all()
+        b = add.accumulate(a) #default axis is 0
+        assert (b == [[0, 1], [2, 4]]).all()
+        # dtype
+        a = arange(0, 3, 0.5).reshape(2, 3)
+        b = add.accumulate(a, dtype=int, axis=1)
+        print b
+        assert (b == [[0, 0, 1], [1, 3, 5]]).all()
+        assert b.dtype == int

From noreply at buildbot.pypy.org  Mon Oct 14 22:00:53 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 14 Oct 2013 22:00:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: implement accumulate and remove unused
 argument to reduce()
Message-ID: <20131014200053.C54151C067F@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67368:09061c91055f
Date: 2013-10-14 18:51 +0300
http://bitbucket.org/pypy/pypy/changeset/09061c91055f/

Log:	implement accumulate and remove unused argument to reduce()

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -850,7 +850,7 @@
             else:
                 out = w_out
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(
-                space, self, True, promote_to_largest, w_axis,
+                space, self, promote_to_largest, w_axis,
                 False, out, w_dtype, cumultative=cumultative)
         return func_with_new_name(impl, "reduce_%s_impl_%d_%d" % (ufunc_name,
                     promote_to_largest, cumultative))
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -75,6 +75,13 @@
                                             'output must be an array'))
         return self.call(space, args_w)
 
+    def descr_accumulate(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None):
+        if space.is_none(w_axis) or w_axis is None:
+            w_axis = space.wrap(0)
+        return self.reduce(space, w_obj, False, #do not promote_to_largest
+                    w_axis, True, #keepdims must be true
+                    w_out, w_dtype, cumultative=True)
+
     @unwrap_spec(skipna=bool, keepdims=bool)
     def descr_reduce(self, space, w_obj, w_axis=None, w_dtype=None,
                      skipna=False, keepdims=False, w_out=None):
@@ -140,10 +147,11 @@
                                                 'output must be an array'))
         else:
             out = w_out
-        return self.reduce(space, w_obj, False, False, w_axis, keepdims, out,
+        promote_to_largest = False
+        return self.reduce(space, w_obj, promote_to_largest, w_axis, keepdims, out,
                            w_dtype)
 
-    def reduce(self, space, w_obj, multidim, promote_to_largest, w_axis,
+    def reduce(self, space, w_obj, promote_to_largest, w_axis,
                keepdims=False, out=None, dtype=None, cumultative=False):
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
@@ -405,6 +413,7 @@
     __repr__ = interp2app(W_Ufunc.descr_repr),
 
     identity = GetSetProperty(W_Ufunc.descr_get_identity),
+    accumulate = interp2app(W_Ufunc.descr_accumulate),
     nin = interp_attrproperty("argcount", cls=W_Ufunc),
 
     reduce = interp2app(W_Ufunc.descr_reduce),

From noreply at buildbot.pypy.org  Mon Oct 14 22:00:54 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 14 Oct 2013 22:00:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix translation,
	fix test for removed numpy.py file
Message-ID: <20131014200054.E2C4D1D22C2@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67369:f25225624539
Date: 2013-10-14 21:27 +0300
http://bitbucket.org/pypy/pypy/changeset/f25225624539/

Log:	fix translation, fix test for removed numpy.py file

diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -78,9 +78,16 @@
     def descr_accumulate(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None):
         if space.is_none(w_axis) or w_axis is None:
             w_axis = space.wrap(0)
+        if space.is_none(w_out):
+            out = None
+        elif not isinstance(w_out, W_NDimArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                                                'output must be an array'))
+        else:
+            out = w_out
         return self.reduce(space, w_obj, False, #do not promote_to_largest
                     w_axis, True, #keepdims must be true
-                    w_out, w_dtype, cumultative=True)
+                    out, w_dtype, cumultative=True)
 
     @unwrap_spec(skipna=bool, keepdims=bool)
     def descr_reduce(self, space, w_obj, w_axis=None, w_dtype=None,
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -97,11 +97,11 @@
 
         # check doubles
         from numpypy import array, nan, zeros, complex128, arange
-        from numpy import isnan
+        from math import isnan
         a = array([nan, 1, 0])
         b = a.copy()
         b.sort()
-        assert (isnan(b) == isnan(a[::-1])).all()
+        assert [isnan(bb) for bb in b] == [isnan(aa) for aa in a[::-1]]
         assert (b[:2] == a[::-1][:2]).all()
 
         # check complex
@@ -110,7 +110,7 @@
         a.imag += [nan, 1, 0, nan, nan, 1, 0, 1, 0]
         b = a.copy()
         b.sort()
-        assert (isnan(b) == isnan(a[::-1])).all()
+        assert [isnan(bb) for bb in b] == [isnan(aa) for aa in a[::-1]]
         assert (b[:4] == a[::-1][:4]).all()
 
         # all c scalar sorts use the same code with different types

From noreply at buildbot.pypy.org  Mon Oct 14 22:00:56 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 14 Oct 2013 22:00:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: add ufunc class to numpy status page
Message-ID: <20131014200056.184911D22CC@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67370:7e94fc81ed0d
Date: 2013-10-14 22:49 +0300
http://bitbucket.org/pypy/pypy/changeset/7e94fc81ed0d/

Log:	add ufunc class to numpy status page

diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py
--- a/pypy/module/micronumpy/tool/numready/main.py
+++ b/pypy/module/micronumpy/tool/numready/main.py
@@ -93,7 +93,7 @@
                 l[i].append(lst[k * lgt + i])
     return l
 
-SPECIAL_NAMES = ["ndarray", "dtype", "generic", "flatiter"]
+SPECIAL_NAMES = ["ndarray", "dtype", "generic", "flatiter", "ufunc"]
 
 def main(argv):
     cpy_items = find_numpy_items("/usr/bin/python")

From noreply at buildbot.pypy.org  Mon Oct 14 22:00:57 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 14 Oct 2013 22:00:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: remove convenience file that conflicts
 with site-packages installed numpy
Message-ID: <20131014200057.3F4741D22D0@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67371:0e97c831046d
Date: 2013-10-14 06:40 +0300
http://bitbucket.org/pypy/pypy/changeset/0e97c831046d/

Log:	remove convenience file that conflicts with site-packages installed
	numpy

diff --git a/lib_pypy/numpy.py b/lib_pypy/numpy.py
deleted file mode 100644
--- a/lib_pypy/numpy.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import warnings
-import sys
-if 'numpypy' not in sys.modules:
-    warnings.warn(
-        "The 'numpy' module of PyPy is in-development and not complete. "
-        "To avoid this warning, write 'import numpypy as numpy'. ",
-        UserWarning) # XXX is this the best warning type?
-
-from numpypy import *
-import numpypy
-__all__ = numpypy.__all__
-del numpypy

From noreply at buildbot.pypy.org  Mon Oct 14 22:21:40 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 14 Oct 2013 22:21:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: Specialize string decoding based on the
 error function when possible
Message-ID: <20131014202140.97AAB1D22C2@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67372:d3f121b34d7f
Date: 2013-10-14 22:20 +0200
http://bitbucket.org/pypy/pypy/changeset/d3f121b34d7f/

Log:	Specialize string decoding based on the error function when possible

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -161,7 +161,7 @@
         loops = log.loops_by_filename(self.filepath)
         assert len(loops) == 1
         for loop in loops:
-            loop.match_by_id('getattr','''
+            assert loop.match_by_id('getattr','''
             guard_not_invalidated?
             i32 = strlen(p31)
             i34 = int_add(5, i32)
@@ -190,11 +190,11 @@
         assert log.result == main(1000)
         loops = log.loops_by_filename(self.filepath)
         loop, = loops
-        loop.match_by_id('callone', '''
+        assert loop.match_by_id('callone', '''
             p114 = call(ConstClass(ll_lower__rpy_stringPtr), p113, descr=<Callr . r EF=3>)
             guard_no_exception(descr=...)
             ''')
-        loop.match_by_id('calltwo', '')    # nothing
+        assert loop.match_by_id('calltwo', '')    # nothing
 
     def test_move_method_call_out_of_loop(self):
         def main(n):
@@ -208,7 +208,7 @@
         assert log.result == main(1000)
         loops = log.loops_by_filename(self.filepath)
         loop, = loops
-        loop.match_by_id('callone', '')    # nothing
+        assert loop.match_by_id('callone', '')    # nothing
 
     def test_lookup_codec(self):
         log = self.run("""
@@ -220,7 +220,7 @@
             return i
         """, [1000])
         loop, = log.loops_by_filename(self.filepath)
-        loop.match("""
+        assert loop.match("""
         i45 = int_lt(i43, i26)
         guard_true(i45, descr=...)
         i46 = int_add(i43, 1)
@@ -229,3 +229,25 @@
         --TICK--
         jump(..., descr=...)
         """)
+
+    def test_decode_ascii(self):
+        log = self.run("""
+        def main(n):
+            for i in xrange(n):
+                unicode('abc')
+            return i
+        """, [1000])
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+        i49 = int_lt(i47, i24)
+        guard_true(i49, descr=...)
+        i50 = int_add(i47, 1)
+        setfield_gc(p15, i50, descr=<FieldS pypy.module.__builtin__.functional.W_XRangeIterator.inst_current 8>)
+        guard_not_invalidated(descr=...)
+        p52 = call(ConstClass(str_decode_ascii__raise_unicode_exception_decode), ConstPtr(ptr38), 3, 1, descr=<Callr 8 rii EF=4>)
+        guard_no_exception(descr=...)
+        p53 = getfield_gc_pure(p52, descr=<FieldP tuple2.item0 8>)
+        guard_nonnull(p53, descr=...)
+        --TICK--
+        jump(..., descr=...)
+        """)
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -967,6 +967,8 @@
     return result.build(), pos
 
 
+# Specialize on the errorhandler when it's a constant
+ at specialize.arg_or_var(4)
 def str_decode_ascii(s, size, errors, final=False,
                      errorhandler=None):
     if errorhandler is None:

From noreply at buildbot.pypy.org  Mon Oct 14 22:25:37 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 14 Oct 2013 22:25:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: This operation is removed nowadays
Message-ID: <20131014202537.B0E031D22C2@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67373:7782ff297bf4
Date: 2013-10-14 22:25 +0200
http://bitbucket.org/pypy/pypy/changeset/7782ff297bf4/

Log:	This operation is removed nowadays

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -122,7 +122,6 @@
             i26 = int_gt(i25, 23)
             guard_false(i26, descr=...)
             strsetitem(p21, i10, 32)
-            i29 = int_add(i10, 1)
             i30 = int_add(i10, i25)
             i31 = int_gt(i30, 23)
             guard_false(i31, descr=...)

From noreply at buildbot.pypy.org  Mon Oct 14 22:30:53 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 14 Oct 2013 22:30:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: test, implement __long__
Message-ID: <20131014203053.15C551D22C2@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67374:b40380d0b61e
Date: 2013-10-14 23:28 +0300
http://bitbucket.org/pypy/pypy/changeset/b40380d0b61e/

Log:	test, implement __long__

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -136,6 +136,11 @@
         assert isinstance(box, W_LongBox)
         return space.wrap(box.value)
 
+    def descr_long(self, space):
+        box = self.convert_to(W_Int64Box._get_dtype(space))
+        assert isinstance(box, W_Int64Box)
+        return space.wrap(box.value)
+
     def descr_float(self, space):
         box = self.convert_to(W_Float64Box._get_dtype(space))
         assert isinstance(box, W_Float64Box)
@@ -470,6 +475,7 @@
     __repr__ = interp2app(W_GenericBox.descr_str),
     __format__ = interp2app(W_GenericBox.descr_format),
     __int__ = interp2app(W_GenericBox.descr_int),
+    __long__ = interp2app(W_GenericBox.descr_long),
     __float__ = interp2app(W_GenericBox.descr_float),
     __nonzero__ = interp2app(W_GenericBox.descr_nonzero),
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -894,6 +894,15 @@
             return space.int(self.descr_getitem(space, space.wrap(0)))
         raise OperationError(space.w_TypeError, space.wrap("only length-1 arrays can be converted to Python scalars"))
 
+    def descr_long(self, space):
+        shape = self.get_shape()
+        if len(shape) == 0:
+            assert isinstance(self.implementation, scalar.Scalar)
+            return space.long(space.wrap(self.implementation.get_scalar_value()))
+        if shape == [1]:
+            return space.int(self.descr_getitem(space, space.wrap(0)))
+        raise OperationError(space.w_TypeError, space.wrap("only length-1 arrays can be converted to Python scalars"))
+
     def descr_float(self, space):
         shape = self.get_shape()
         if len(shape) == 0:
@@ -1020,6 +1029,7 @@
     __repr__ = interp2app(W_NDimArray.descr_repr),
     __str__ = interp2app(W_NDimArray.descr_str),
     __int__ = interp2app(W_NDimArray.descr_int),
+    __long__ = interp2app(W_NDimArray.descr_long),
     __float__ = interp2app(W_NDimArray.descr_float),
 
     __pos__ = interp2app(W_NDimArray.descr_pos),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1896,6 +1896,15 @@
         b = array([1, 2, 3, 4])
         assert (a == b) == False
 
+    def test__long__(self):
+        from numpypy import array
+        assert long(array(1)) == 1
+        assert long(array([1])) == 1
+        assert isinstance(long(array([1])), long)
+        assert isinstance(long(array([1, 2][0])), long)
+        assert raises(TypeError, "long(array([1, 2]))")
+        assert long(array([1.5])) == 1
+
     def test__int__(self):
         from numpypy import array
         assert int(array(1)) == 1

From noreply at buildbot.pypy.org  Tue Oct 15 00:09:14 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 15 Oct 2013 00:09:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: Miniscule optimization to mapdict.
Message-ID: <20131014220914.64CE81C01B0@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67375:b8d22f47402d
Date: 2013-10-15 00:06 +0200
http://bitbucket.org/pypy/pypy/changeset/b8d22f47402d/

Log:	Miniscule optimization to mapdict.

	n is a constant here because it comes from the closure so subtract 1
	from it before doing the arithmetic, this way we generate len(...) +
	CONSTANT instead of len(...) + CONSTANT - 1

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -556,7 +556,7 @@
 
         def _mapdict_storage_length(self):
             if self._has_storage_list():
-                return len(self._mapdict_get_storage_list()) + n - 1
+                return len(self._mapdict_get_storage_list()) + (n - 1)
             return n
 
         def _set_mapdict_storage_and_map(self, storage, map):

From noreply at buildbot.pypy.org  Tue Oct 15 00:09:15 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 15 Oct 2013 00:09:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20131014220915.9B4C71C01B0@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67376:261f84f41858
Date: 2013-10-15 00:08 +0200
http://bitbucket.org/pypy/pypy/changeset/261f84f41858/

Log:	merged upstream

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -136,6 +136,11 @@
         assert isinstance(box, W_LongBox)
         return space.wrap(box.value)
 
+    def descr_long(self, space):
+        box = self.convert_to(W_Int64Box._get_dtype(space))
+        assert isinstance(box, W_Int64Box)
+        return space.wrap(box.value)
+
     def descr_float(self, space):
         box = self.convert_to(W_Float64Box._get_dtype(space))
         assert isinstance(box, W_Float64Box)
@@ -470,6 +475,7 @@
     __repr__ = interp2app(W_GenericBox.descr_str),
     __format__ = interp2app(W_GenericBox.descr_format),
     __int__ = interp2app(W_GenericBox.descr_int),
+    __long__ = interp2app(W_GenericBox.descr_long),
     __float__ = interp2app(W_GenericBox.descr_float),
     __nonzero__ = interp2app(W_GenericBox.descr_nonzero),
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -894,6 +894,15 @@
             return space.int(self.descr_getitem(space, space.wrap(0)))
         raise OperationError(space.w_TypeError, space.wrap("only length-1 arrays can be converted to Python scalars"))
 
+    def descr_long(self, space):
+        shape = self.get_shape()
+        if len(shape) == 0:
+            assert isinstance(self.implementation, scalar.Scalar)
+            return space.long(space.wrap(self.implementation.get_scalar_value()))
+        if shape == [1]:
+            return space.int(self.descr_getitem(space, space.wrap(0)))
+        raise OperationError(space.w_TypeError, space.wrap("only length-1 arrays can be converted to Python scalars"))
+
     def descr_float(self, space):
         shape = self.get_shape()
         if len(shape) == 0:
@@ -1020,6 +1029,7 @@
     __repr__ = interp2app(W_NDimArray.descr_repr),
     __str__ = interp2app(W_NDimArray.descr_str),
     __int__ = interp2app(W_NDimArray.descr_int),
+    __long__ = interp2app(W_NDimArray.descr_long),
     __float__ = interp2app(W_NDimArray.descr_float),
 
     __pos__ = interp2app(W_NDimArray.descr_pos),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1896,6 +1896,15 @@
         b = array([1, 2, 3, 4])
         assert (a == b) == False
 
+    def test__long__(self):
+        from numpypy import array
+        assert long(array(1)) == 1
+        assert long(array([1])) == 1
+        assert isinstance(long(array([1])), long)
+        assert isinstance(long(array([1, 2][0])), long)
+        assert raises(TypeError, "long(array([1, 2]))")
+        assert long(array([1.5])) == 1
+
     def test__int__(self):
         from numpypy import array
         assert int(array(1)) == 1

From noreply at buildbot.pypy.org  Tue Oct 15 01:17:57 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 01:17:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: make this numpy import relative
Message-ID: <20131014231757.59E041C02C2@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67377:f84ecdace85b
Date: 2013-10-14 19:17 -0400
http://bitbucket.org/pypy/pypy/changeset/f84ecdace85b/

Log:	make this numpy import relative

diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -606,7 +606,7 @@
            [ 0.,  0.,  1.]])
 
     """
-    from numpy import eye
+    from .. import eye
     return eye(n, dtype=dtype)
 
 Inf = inf = infty = Infinity = PINF

From noreply at buildbot.pypy.org  Tue Oct 15 05:58:33 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 15 Oct 2013 05:58:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: fixes for removed numpy.py
Message-ID: <20131015035833.A8A311C02C2@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67378:f320ec7a85b5
Date: 2013-10-15 06:57 +0300
http://bitbucket.org/pypy/pypy/changeset/f320ec7a85b5/

Log:	fixes for removed numpy.py

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -28,9 +28,10 @@
         ...
 
     """
-    import numpy
-    if getattr(numpy, 'show_config', None) is None:
-        # running from numpy source directory
+    try:
+        import numpy
+    except:
+        # running from pypy source directory
         head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
         return os.path.join(head, '../include')
     else:
@@ -44,7 +45,6 @@
 __all__ = ['__version__', 'get_include']
 __all__ += core.__all__
 __all__ += lib.__all__
-
 #import sys
 #sys.modules.setdefault('numpy', sys.modules['numpypy'])
 
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -1,11 +1,11 @@
-######################################################################    
+######################################################################
 # This is a copy of numpy/core/fromnumeric.py modified for numpypy
 ######################################################################
-# Each name in __all__ was a function in  'numeric' that is now 
+# Each name in __all__ was a function in  'numeric' that is now
 # a method in 'numpy'.
 # When the corresponding method is added to numpypy BaseArray
-# each function should be added as a module function 
-# at the applevel 
+# each function should be added as a module function
+# at the applevel
 # This can be as simple as doing the following
 #
 # def func(a, ...):
@@ -15,8 +15,8 @@
 #
 ######################################################################
 
-import numpypy
 import _numpypy
+from _numpypy.multiarray import array as numpyarray
 
 # Module containing non-deprecated functions borrowed from Numeric.
 __docformat__ = "restructuredtext en"
@@ -152,7 +152,7 @@
     """
     assert order == 'C'
     if not hasattr(a, 'reshape'):
-       a = numpypy.array(a)
+       a = numpyarray(a)
     return a.reshape(newshape)
 
 
@@ -457,7 +457,7 @@
     if axes is not None:
         raise NotImplementedError('No "axes" arg yet.')
     if not hasattr(a, 'T'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.T
 
 def sort(a, axis=-1, kind='quicksort', order=None):
@@ -695,7 +695,7 @@
     """
     assert axis is None
     if not hasattr(a, 'argmax'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.argmax()
 
 
@@ -711,7 +711,7 @@
     """
     assert axis is None
     if not hasattr(a, 'argmin'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.argmin()
 
 
@@ -1057,7 +1057,7 @@
 
     """
     if not hasattr(a, 'ravel'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.ravel(order=order)
 
 def nonzero(a):
@@ -1181,7 +1181,7 @@
 
     """
     if not hasattr(a, 'shape'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.shape
 
 
@@ -1298,7 +1298,7 @@
 
     """
     if not hasattr(a, 'clip'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.clip(a_min, a_max, out=out)
 
 
@@ -1370,7 +1370,7 @@
     """
     assert dtype is None
     if not hasattr(a, "sum"):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.sum(axis=axis, out=out)
 
 
@@ -1400,7 +1400,7 @@
     assert axis is None
     assert out is None
     if not hasattr(a, 'any'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.any()
 
 
@@ -1416,7 +1416,7 @@
     assert axis is None
     assert out is None
     if not hasattr(a, 'all'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.all()
 
 def any(a,axis=None, out=None):
@@ -1486,7 +1486,7 @@
     assert axis is None
     assert out is None
     if not hasattr(a, 'any'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.any()
 
 
@@ -1550,7 +1550,7 @@
     assert axis is None
     assert out is None
     if not hasattr(a, 'all'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.all()
 
 
@@ -1729,9 +1729,9 @@
 
     """
     if not hasattr(a, "max"):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     if a.size < 1:
-        return numpypy.array([])
+        return numpyarray([])
     return a.max(axis=axis, out=out)
 
 
@@ -1791,9 +1791,9 @@
 
     """
     if not hasattr(a, 'min'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     if a.size < 1:
-        return numpypy.array([])
+        return numpyarray([])
     return a.min(axis=axis, out=out)
 
 def alen(a):
@@ -1824,7 +1824,7 @@
 
     """
     if not hasattr(a, 'shape'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.shape[0]
 
 
@@ -2000,7 +2000,7 @@
 
     """
     if not hasattr(a, 'ndim'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.ndim
 
 
@@ -2045,7 +2045,7 @@
 
     """
     if not hasattr(a, 'ndim'):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.ndim
 
 
@@ -2243,7 +2243,7 @@
     assert dtype is None
     assert out is None
     if not hasattr(a, "mean"):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.mean(axis=axis)
 
 
@@ -2337,7 +2337,7 @@
     assert out is None
     assert ddof == 0
     if not hasattr(a, "std"):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.std(axis=axis)
 
 
@@ -2433,5 +2433,6 @@
     assert out is None
     assert ddof == 0
     if not hasattr(a, "var"):
-        a = numpypy.array(a)
+        a = numpyarray(a)
     return a.var(axis=axis)
+
diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -1,5 +1,5 @@
 from _numpypy.numerictypes import *
-import numpypy
+from _numpypy.multiarray import dtype
 
 def issubclass_(arg1, arg2):
     """
@@ -66,10 +66,10 @@
 
     """
     if issubclass_(arg2, generic):
-        return issubclass(numpypy.dtype(arg1).type, arg2)
-    mro = numpypy.dtype(arg2).type.mro()
+        return issubclass(dtype(arg1).type, arg2)
+    mro = dtype(arg2).type.mro()
     if len(mro) > 1:
         val = mro[1]
     else:
         val = mro[0]
-    return issubclass(numpypy.dtype(arg1).type, val)
+    return issubclass(dtype(arg1).type, val)
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -4,7 +4,7 @@
 class AppTestFromNumeric(BaseNumpyAppTest):
     def test_argmax(self):
         # tests taken from numpy/core/fromnumeric.py docstring
-        from numpypy import array, arange, argmax
+        from numpypy import arange, argmax
         a = arange(6).reshape((2,3))
         assert argmax(a) == 5
         # assert (argmax(a, axis=0) == array([1, 1, 1])).all()
@@ -15,7 +15,7 @@
 
     def test_argmin(self):
         # tests adapted from test_argmax
-        from numpypy import array, arange, argmin
+        from numpypy import arange, argmin
         a = arange(6).reshape((2,3))
         assert argmin(a) == 0
         #assert (argmin(a, axis=0) == array([0, 0, 0])).all()
@@ -26,7 +26,7 @@
 
     def test_shape(self):
         # tests taken from numpy/core/fromnumeric.py docstring
-        from numpypy import array, identity, shape
+        from numpypy import identity, shape
         assert shape(identity(3)) == (3, 3)
         assert shape([[1, 2]]) == (1, 2)
         assert shape([0]) ==  (1,)
@@ -50,7 +50,7 @@
 
     def test_sum(self):
         # tests taken from numpy/core/fromnumeric.py docstring
-        from numpypy import array, sum, ones, zeros
+        from numpypy import sum, ones, zeros, array
         assert sum([0.5, 1.5])== 2.0
         assert sum([[0, 1], [0, 5]]) == 6
         # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1
@@ -175,7 +175,7 @@
         assert reshape(a, (1, 1, -1)).shape == (1, 1, 105)
         assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1)
 
-    def test_transpose(self):   
+    def test_transpose(self):
         from numpypy import arange, array, transpose, ones
         x = arange(4).reshape((2,2))
         assert (transpose(x) == array([[0, 2],[1, 3]])).all()
@@ -189,5 +189,5 @@
         x = array([[1,2,3]])
         assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
         x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
-        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]],
                                           [[1, 5], [3, 7]]])).all()
diff --git a/pypy/module/test_lib_pypy/numpypy/test_numpy.py b/pypy/module/test_lib_pypy/numpypy/test_numpy.py
--- a/pypy/module/test_lib_pypy/numpypy/test_numpy.py
+++ b/pypy/module/test_lib_pypy/numpypy/test_numpy.py
@@ -2,47 +2,6 @@
 import py, sys
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-class AppTestNumpyImport1(object):
-    spaceconfig = dict(usemodules=['micronumpy'])
-
-    @classmethod
-    def setup_class(cls):
-        if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("pypy only test")
-
-    def test_imports_no_warning(self):
-        from warnings import catch_warnings
-        with catch_warnings(record=True) as w:
-            import numpypy
-            import numpy
-            assert len(w) == 0
-            import numpy
-            assert len(w) == 0
-
-class AppTestNumpyImport2(object):
-    spaceconfig = dict(usemodules=['micronumpy'])
-
-    @classmethod
-    def setup_class(cls):
-        if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("pypy only test")
-
-    def test_imports_with_warning(self):
-        import sys
-        from warnings import catch_warnings
-        # XXX why are numpypy and numpy modules already imported?
-        mods = [d for d in sys.modules.keys() if d.find('numpy') >= 0]
-        if mods:
-            skip('%s already imported' % mods)
-
-        with catch_warnings(record=True) as w:
-            import numpy
-            msg = w[0].message
-            assert msg.message.startswith(
-                "The 'numpy' module of PyPy is in-development")
-            import numpy
-            assert len(w) == 1
-
 class AppTestNumpy(BaseNumpyAppTest):
     def test_min_max_after_import(self):
         import __builtin__
@@ -95,15 +54,15 @@
         assert math.isnan(numpypy.nan)
 
     def test___all__(self):
-        import numpy
-        assert '__all__' in dir(numpy)
-        assert 'numpypy' not in dir(numpy)
+        import numpypy
+        assert '__all__' in dir(numpypy)
+        assert 'numpypy' not in dir(numpypy)
 
     def test_get_include(self):
         import sys
         if not hasattr(sys, 'pypy_translation_info'):
             skip("pypy white-box test")
-        import numpy, os
-        assert 'get_include' in dir(numpy)
-        path = numpy.get_include()
+        import numpypy, os
+        assert 'get_include' in dir(numpypy)
+        path = numpypy.get_include()
         assert os.path.exists(path + '/numpy/arrayobject.h')

From noreply at buildbot.pypy.org  Tue Oct 15 07:28:48 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 15 Oct 2013 07:28:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: Account for 32 vs 64 bit
Message-ID: <20131015052848.D38331C0205@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67379:28869df61bb8
Date: 2013-10-15 07:28 +0200
http://bitbucket.org/pypy/pypy/changeset/28869df61bb8/

Log:	Account for 32 vs 64 bit

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -243,7 +243,7 @@
         i50 = int_add(i47, 1)
         setfield_gc(p15, i50, descr=<FieldS pypy.module.__builtin__.functional.W_XRangeIterator.inst_current 8>)
         guard_not_invalidated(descr=...)
-        p52 = call(ConstClass(str_decode_ascii__raise_unicode_exception_decode), ConstPtr(ptr38), 3, 1, descr=<Callr 8 rii EF=4>)
+        p52 = call(ConstClass(str_decode_ascii__raise_unicode_exception_decode), ConstPtr(ptr38), 3, 1, descr=<Callr . rii EF=4>)
         guard_no_exception(descr=...)
         p53 = getfield_gc_pure(p52, descr=<FieldP tuple2.item0 8>)
         guard_nonnull(p53, descr=...)

From noreply at buildbot.pypy.org  Tue Oct 15 09:14:41 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 09:14:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: clean up some more numpy imports/work
 towards compat with upstream numpy code
Message-ID: <20131015071441.932B21C01B0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67380:c65912c9d433
Date: 2013-10-15 01:16 -0400
http://bitbucket.org/pypy/pypy/changeset/c65912c9d433/

Log:	clean up some more numpy imports/work towards compat with upstream
	numpy code

diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
--- a/lib_pypy/numpypy/core/__init__.py
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -1,12 +1,16 @@
-import numeric
-from numeric import *
-import fromnumeric
-from fromnumeric import *
-import shape_base
-from shape_base import *
+from __future__ import division, absolute_import, print_function
 
-from fromnumeric import amax as max, amin as min
-from numeric import absolute as abs
+from . import multiarray
+from . import umath
+from . import numeric
+from .numeric import *
+from . import fromnumeric
+from .fromnumeric import *
+from . import shape_base
+from .shape_base import *
+
+from .fromnumeric import amax as max, amin as min
+from .numeric import absolute as abs
 
 __all__ = []
 __all__ += numeric.__all__
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -1,3 +1,5 @@
+from __future__ import division, absolute_import, print_function
+
 __all__ = [
            'newaxis', 'ufunc',
            'asarray', 'asanyarray', 'base_repr',
@@ -8,14 +10,14 @@
           ]
 
 import sys
-import multiarray
-from multiarray import *
+from . import multiarray
+from .multiarray import *
 del set_string_function
 del typeinfo
-import umath
-from umath import *
-import numerictypes
-from numerictypes import *
+from . import umath
+from .umath import *
+from . import numerictypes
+from .numerictypes import *
 
 def extend_all(module):
     adict = {}
@@ -41,6 +43,76 @@
 def seterr(**args):
     return args
 
+def asarray(a, dtype=None, order=None):
+    """
+    Convert the input to an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data, in any form that can be converted to an array.  This
+        includes lists, lists of tuples, tuples, tuples of tuples, tuples
+        of lists and ndarrays.
+    dtype : data-type, optional
+        By default, the data-type is inferred from the input data.
+    order : {'C', 'F'}, optional
+        Whether to use row-major ('C') or column-major ('F' for FORTRAN)
+        memory representation.  Defaults to 'C'.
+
+    Returns
+    -------
+    out : ndarray
+        Array interpretation of `a`.  No copy is performed if the input
+        is already an ndarray.  If `a` is a subclass of ndarray, a base
+        class ndarray is returned.
+
+    See Also
+    --------
+    asanyarray : Similar function which passes through subclasses.
+    ascontiguousarray : Convert input to a contiguous array.
+    asfarray : Convert input to a floating point ndarray.
+    asfortranarray : Convert input to an ndarray with column-major
+                     memory order.
+    asarray_chkfinite : Similar function which checks input for NaNs and Infs.
+    fromiter : Create an array from an iterator.
+    fromfunction : Construct an array by executing a function on grid
+                   positions.
+
+    Examples
+    --------
+    Convert a list into an array:
+
+    >>> a = [1, 2]
+    >>> np.asarray(a)
+    array([1, 2])
+
+    Existing arrays are not copied:
+
+    >>> a = np.array([1, 2])
+    >>> np.asarray(a) is a
+    True
+
+    If `dtype` is set, array is copied only if dtype does not match:
+
+    >>> a = np.array([1, 2], dtype=np.float32)
+    >>> np.asarray(a, dtype=np.float32) is a
+    True
+    >>> np.asarray(a, dtype=np.float64) is a
+    False
+
+    Contrary to `asanyarray`, ndarray subclasses are not passed through:
+
+    >>> issubclass(np.matrix, np.ndarray)
+    True
+    >>> a = np.matrix([[1, 2]])
+    >>> np.asarray(a) is a
+    False
+    >>> np.asanyarray(a) is a
+    True
+
+    """
+    return array(a, dtype, copy=False, order=order)
+
 def asanyarray(a, dtype=None, order=None):
     """
     Convert the input to an ndarray, but pass ndarray subclasses through.
@@ -148,7 +220,7 @@
 
 
 #Use numarray's printing function
-from arrayprint import array2string
+from .arrayprint import array2string
 
 _typelessdata = [int_, float_]#, complex_]
 # XXX
@@ -381,76 +453,6 @@
         return False
     return bool((a1 == a2).all())
 
-def asarray(a, dtype=None, order=None):
-    """
-    Convert the input to an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input data, in any form that can be converted to an array.  This
-        includes lists, lists of tuples, tuples, tuples of tuples, tuples
-        of lists and ndarrays.
-    dtype : data-type, optional
-        By default, the data-type is inferred from the input data.
-    order : {'C', 'F'}, optional
-        Whether to use row-major ('C') or column-major ('F' for FORTRAN)
-        memory representation.  Defaults to 'C'.
-
-    Returns
-    -------
-    out : ndarray
-        Array interpretation of `a`.  No copy is performed if the input
-        is already an ndarray.  If `a` is a subclass of ndarray, a base
-        class ndarray is returned.
-
-    See Also
-    --------
-    asanyarray : Similar function which passes through subclasses.
-    ascontiguousarray : Convert input to a contiguous array.
-    asfarray : Convert input to a floating point ndarray.
-    asfortranarray : Convert input to an ndarray with column-major
-                     memory order.
-    asarray_chkfinite : Similar function which checks input for NaNs and Infs.
-    fromiter : Create an array from an iterator.
-    fromfunction : Construct an array by executing a function on grid
-                   positions.
-
-    Examples
-    --------
-    Convert a list into an array:
-
-    >>> a = [1, 2]
-    >>> np.asarray(a)
-    array([1, 2])
-
-    Existing arrays are not copied:
-
-    >>> a = np.array([1, 2])
-    >>> np.asarray(a) is a
-    True
-
-    If `dtype` is set, array is copied only if dtype does not match:
-
-    >>> a = np.array([1, 2], dtype=np.float32)
-    >>> np.asarray(a, dtype=np.float32) is a
-    True
-    >>> np.asarray(a, dtype=np.float64) is a
-    False
-
-    Contrary to `asanyarray`, ndarray subclasses are not passed through:
-
-    >>> issubclass(np.matrix, np.ndarray)
-    True
-    >>> a = np.matrix([[1, 2]])
-    >>> np.asarray(a) is a
-    False
-    >>> np.asanyarray(a) is a
-    True
-
-    """
-    return array(a, dtype, copy=False, order=order)
-
 def outer(a,b):
     """
     Compute the outer product of two vectors.
@@ -614,6 +616,6 @@
 False_ = bool_(False)
 True_ = bool_(True)
 
-import fromnumeric
-from fromnumeric import *
+from . import fromnumeric
+from .fromnumeric import *
 extend_all(fromnumeric)
diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -1,5 +1,6 @@
 from _numpypy.numerictypes import *
-from _numpypy.multiarray import dtype
+
+from .multiarray import dtype
 
 def issubclass_(arg1, arg2):
     """

From noreply at buildbot.pypy.org  Tue Oct 15 09:14:42 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 09:14:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix behavior of numpypy reduce wrt
	zero-sized dims
Message-ID: <20131015071442.F0F5D1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67381:fee031ae7721
Date: 2013-10-15 02:40 -0400
http://bitbucket.org/pypy/pypy/changeset/fee031ae7721/

Log:	fix behavior of numpypy reduce wrt zero-sized dims

diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -76,7 +76,7 @@
         return self.call(space, args_w)
 
     def descr_accumulate(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None):
-        if space.is_none(w_axis) or w_axis is None:
+        if space.is_none(w_axis):
             w_axis = space.wrap(0)
         if space.is_none(w_out):
             out = None
@@ -186,9 +186,12 @@
                     promote_to_largest=promote_to_largest,
                     promote_bools=True
                 )
-        if self.identity is None and size == 0:
-            raise operationerrfmt(space.w_ValueError, "zero-size array to "
-                    "%s.reduce without identity", self.name)
+        if self.identity is None:
+            for i in range(shapelen):
+                if space.is_none(w_axis) or i == axis:
+                    if obj_shape[i] == 0:
+                        raise operationerrfmt(space.w_ValueError, "zero-size array to "
+                                "%s.reduce without identity", self.name)
         if shapelen > 1 and axis < shapelen:
             temp = None
             if cumultative:
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -657,11 +657,15 @@
             assert b[i] == math.degrees(a[i])
 
     def test_reduce_errors(self):
-        from numpypy import sin, add
+        from numpypy import sin, add, maximum, zeros
 
         raises(ValueError, sin.reduce, [1, 2, 3])
         assert add.reduce(1) == 1
 
+        assert list(maximum.reduce(zeros((2, 0)), axis=0)) == []
+        raises(ValueError, maximum.reduce, zeros((2, 0)), axis=None)
+        raises(ValueError, maximum.reduce, zeros((2, 0)), axis=1)
+
     def test_reduce_1d(self):
         from numpypy import add, maximum, less
 

From noreply at buildbot.pypy.org  Tue Oct 15 09:14:44 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 09:14:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: import a new fromnumeric/shape_base
	from numpy
Message-ID: <20131015071444.4A0621C01B0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67382:413dbfc0f55b
Date: 2013-10-15 02:42 -0400
http://bitbucket.org/pypy/pypy/changeset/413dbfc0f55b/

Log:	import a new fromnumeric/shape_base from numpy

diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
--- a/lib_pypy/numpypy/core/__init__.py
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -9,7 +9,8 @@
 from . import shape_base
 from .shape_base import *
 
-from .fromnumeric import amax as max, amin as min
+from .fromnumeric import amax as max, amin as min, \
+    round_ as round
 from .numeric import absolute as abs
 
 __all__ = []
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -1,36 +1,53 @@
 ######################################################################
 # This is a copy of numpy/core/fromnumeric.py modified for numpypy
 ######################################################################
-# Each name in __all__ was a function in  'numeric' that is now
-# a method in 'numpy'.
-# When the corresponding method is added to numpypy BaseArray
-# each function should be added as a module function
-# at the applevel
-# This can be as simple as doing the following
-#
-# def func(a, ...):
-#     if not hasattr(a, 'func')
-#         a = numpypy.array(a)
-#     return a.func(...)
-#
-######################################################################
-
-import _numpypy
-from _numpypy.multiarray import array as numpyarray
-
-# Module containing non-deprecated functions borrowed from Numeric.
-__docformat__ = "restructuredtext en"
+"""Module containing non-deprecated functions borrowed from Numeric.
+
+"""
+from __future__ import division, absolute_import, print_function
+
+import types
+
+from . import multiarray as mu
+from . import umath as um
+from . import numerictypes as nt
+from .numeric import asarray, array, asanyarray, concatenate
+from . import _methods
+
+
+# functions that are methods
+__all__ = [
+        'alen', 'all', 'alltrue', 'amax', 'amin', 'any', 'argmax',
+        'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip',
+        'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean',
+        'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put',
+        'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_',
+        'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze',
+        'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var',
+        ]
+
+
+try:
+    _gentype = types.GeneratorType
+except AttributeError:
+    _gentype = type(None)
+
+# save away Python sum
+_sum_ = sum
 
 # functions that are now methods
-__all__ = ['take', 'reshape', 'choose', 'repeat', 'put',
-           'swapaxes', 'transpose', 'sort', 'argsort', 'argmax', 'argmin',
-           'searchsorted', 'alen',
-           'resize', 'diagonal', 'trace', 'ravel', 'nonzero', 'shape',
-           'compress', 'clip', 'sum', 'product', 'prod', 'sometrue', 'alltrue',
-           'any', 'all', 'cumsum', 'cumproduct', 'cumprod', 'ptp', 'ndim',
-           'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze',
-           'amax', 'amin',
-          ]
+def _wrapit(obj, method, *args, **kwds):
+    try:
+        wrap = obj.__array_wrap__
+    except AttributeError:
+        wrap = None
+    result = getattr(asarray(obj), method)(*args, **kwds)
+    if wrap:
+        if not isinstance(result, mu.ndarray):
+            result = asarray(result)
+        result = wrap(result)
+    return result
+
 
 def take(a, indices, axis=None, out=None, mode='raise'):
     """
@@ -46,6 +63,10 @@
         The source array.
     indices : array_like
         The indices of the values to extract.
+
+        .. versionadded:: 1.8.0
+
+        Also allow scalars for indices.
     axis : int, optional
         The axis over which to select values. By default, the flattened
         input array is used.
@@ -85,8 +106,17 @@
     >>> a[indices]
     array([4, 3, 6])
 
+    If `indices` is not one dimensional, the output also has these dimensions.
+
+    >>> np.take(a, [[0, 1], [2, 3]])
+    array([[4, 3],
+           [5, 7]])
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        take = a.take
+    except AttributeError:
+        return _wrapit(a, 'take', indices, axis, out, mode)
+    return take(indices, axis, out, mode)
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -104,16 +134,23 @@
         One shape dimension can be -1. In this case, the value is inferred
         from the length of the array and remaining dimensions.
     order : {'C', 'F', 'A'}, optional
-        Determines whether the array data should be viewed as in C
-        (row-major) order, FORTRAN (column-major) order, or the C/FORTRAN
-        order should be preserved.
+        Read the elements of `a` using this index order, and place the elements
+        into the reshaped array using this index order.  'C' means to
+        read / write the elements using C-like index order, with the last axis index
+        changing fastest, back to the first axis index changing slowest.  'F'
+        means to read / write the elements using Fortran-like index order, with
+        the first index changing fastest, and the last index changing slowest.
+        Note that the 'C' and 'F' options take no account of the memory layout
+        of the underlying array, and only refer to the order of indexing.  'A'
+        means to read / write the elements in Fortran-like index order if `a` is
+        Fortran *contiguous* in memory, C-like order otherwise.
 
     Returns
     -------
     reshaped_array : ndarray
         This will be a new view object if possible; otherwise, it will
-        be a copy.
-
+        be a copy.  Note there is no guarantee of the *memory layout* (C- or
+        Fortran- contiguous) of the returned array.
 
     See Also
     --------
@@ -121,7 +158,6 @@
 
     Notes
     -----
-
     It is not always possible to change the shape of an array without
     copying the data. If you want an error to be raise if the data is copied,
     you should assign the new shape to the shape attribute of the array::
@@ -129,12 +165,39 @@
      >>> a = np.zeros((10, 2))
      # A transpose make the array non-contiguous
      >>> b = a.T
-     # Taking a view makes it possible to modify the shape without modiying the
+     # Taking a view makes it possible to modify the shape without modifying the
      # initial object.
      >>> c = b.view()
      >>> c.shape = (20)
      AttributeError: incompatible shape for a non-contiguous array
 
+    The `order` keyword gives the index ordering both for *fetching* the values
+    from `a`, and then *placing* the values into the output array.  For example,
+    let's say you have an array:
+
+    >>> a = np.arange(6).reshape((3, 2))
+    >>> a
+    array([[0, 1],
+           [2, 3],
+           [4, 5]])
+
+    You can think of reshaping as first raveling the array (using the given
+    index order), then inserting the elements from the raveled array into the
+    new array using the same kind of index ordering as was used for the
+    raveling.
+
+    >>> np.reshape(a, (2, 3)) # C-like index ordering
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(a, (2, 3), order='F') # Fortran-like index ordering
+    array([[0, 4, 3],
+           [2, 1, 5]])
+    >>> np.reshape(np.ravel(a, order='F'), (2, 3), order='F')
+    array([[0, 4, 3],
+           [2, 1, 5]])
 
     Examples
     --------
@@ -148,12 +211,13 @@
     array([[1, 2],
            [3, 4],
            [5, 6]])
-
     """
     assert order == 'C'
-    if not hasattr(a, 'reshape'):
-       a = numpyarray(a)
-    return a.reshape(newshape)
+    try:
+        reshape = a.reshape
+    except AttributeError:
+        return _wrapit(a, 'reshape', newshape)
+    return reshape(newshape)
 
 
 def choose(a, choices, out=None, mode='raise'):
@@ -275,7 +339,11 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    return _numpypy.choose(a, choices, out, mode)
+    try:
+        choose = a.choose
+    except AttributeError:
+        return _wrapit(a, 'choose', choices, out=out, mode=mode)
+    return choose(choices, out=out, mode=mode)
 
 
 def repeat(a, repeats, axis=None):
@@ -317,7 +385,11 @@
            [3, 4]])
 
     """
-    return _numpypy.repeat(a, repeats, axis)
+    try:
+        repeat = a.repeat
+    except AttributeError:
+        return _wrapit(a, 'repeat', repeats, axis)
+    return repeat(repeats, axis)
 
 
 def put(a, ind, v, mode='raise'):
@@ -368,7 +440,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return a.put(ind, v, mode)
 
 
 def swapaxes(a, axis1, axis2):
@@ -412,7 +484,10 @@
             [3, 7]]])
 
     """
-    swapaxes = a.swapaxes
+    try:
+        swapaxes = a.swapaxes
+    except AttributeError:
+        return _wrapit(a, 'swapaxes', axis1, axis2)
     return swapaxes(axis1, axis2)
 
 
@@ -456,9 +531,158 @@
     """
     if axes is not None:
         raise NotImplementedError('No "axes" arg yet.')
-    if not hasattr(a, 'T'):
-        a = numpyarray(a)
-    return a.T
+    try:
+        transpose = a.transpose
+    except AttributeError:
+        return _wrapit(a, 'transpose')
+    return transpose()
+
+
+def partition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Return a partitioned copy of an array.
+
+    Creates a copy of the array with its elements rearranged in such a way that
+    the value of the element in kth position is in the position it would be in
+    a sorted array. All elements smaller than the kth element are moved before
+    this element and all equal or greater are moved behind it. The ordering of
+    the elements in the two partitions is undefined.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to be sorted.
+    kth : int or sequence of ints
+        Element index to partition by. The kth value of the element will be in
+        its final sorted position and all smaller elements will be moved before
+        it and all equal or greater elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all elements
+        indexed by kth  of them into their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort. If None, the array is flattened before
+        sorting. The default is -1, which sorts along the last axis.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'.
+    order : list, optional
+        When `a` is a structured array, this argument specifies which fields
+        to compare first, second, and so on.  This list does not need to
+        include all of the fields.
+
+    Returns
+    -------
+    partitioned_array : ndarray
+        Array of the same type and shape as `a`.
+
+    See Also
+    --------
+    ndarray.partition : Method to sort an array in-place.
+    argpartition : Indirect partition.
+    sort : Full sorting
+
+    Notes
+    -----
+    The various selection algorithms are characterized by their average speed,
+    worst case performance, work space size, and whether they are stable. A
+    stable sort keeps items with the same key in the same relative order. The
+    three available algorithms have the following properties:
+
+    ================= ======= ============= ============ =======
+       kind            speed   worst case    work space  stable
+    ================= ======= ============= ============ =======
+    'introselect'        1        O(n)           0         no
+    ================= ======= ============= ============ =======
+
+    All the partition algorithms make temporary copies of the data when
+    partitioning along any but the last axis.  Consequently, partitioning
+    along the last axis is faster and uses less space than partitioning
+    along any other axis.
+
+    The sort order for complex numbers is lexicographic. If both the real
+    and imaginary parts are non-nan then the order is determined by the
+    real parts except when they are equal, in which case the order is
+    determined by the imaginary parts.
+
+    Examples
+    --------
+    >>> a = np.array([3, 4, 2, 1])
+    >>> np.partition(a, 3)
+    array([2, 1, 3, 4])
+
+    >>> np.partition(a, (1, 3))
+    array([1, 2, 3, 4])
+
+    """
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.partition(kth, axis=axis, kind=kind, order=order)
+    return a
+
+
+def argpartition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Perform an indirect partition along the given axis using the algorithm
+    specified by the `kind` keyword. It returns an array of indices of the
+    same shape as `a` that index data along the given axis in partitioned
+    order.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to sort.
+    kth : int or sequence of ints
+        Element index to partition by. The kth element will be in its final
+        sorted position and all smaller elements will be moved before it and
+        all larger elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all of them into
+        their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort.  The default is -1 (the last axis). If None,
+        the flattened array is used.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'
+    order : list, optional
+        When `a` is an array with fields defined, this argument specifies
+        which fields to compare first, second, etc.  Not all fields need be
+        specified.
+
+    Returns
+    -------
+    index_array : ndarray, int
+        Array of indices that partition `a` along the specified axis.
+        In other words, ``a[index_array]`` yields a sorted `a`.
+
+    See Also
+    --------
+    partition : Describes partition algorithms used.
+    ndarray.partition : Inplace partition.
+    argsort : Full indirect sort
+
+    Notes
+    -----
+    See `partition` for notes on the different selection algorithms.
+
+    Examples
+    --------
+    One dimensional array:
+
+    >>> x = np.array([3, 4, 2, 1])
+    >>> x[np.argpartition(x, 3)]
+    array([2, 1, 3, 4])
+    >>> x[np.argpartition(x, (1, 3))]
+    array([1, 2, 3, 4])
+
+    """
+    return a.argpartition(kth, axis, kind=kind, order=order)
+
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -489,6 +713,7 @@
     argsort : Indirect sort.
     lexsort : Indirect stable sort on multiple keys.
     searchsorted : Find elements in a sorted array.
+    partition : Partial sort.
 
     Notes
     -----
@@ -559,7 +784,13 @@
           dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.sort(axis, kind, order)
+    return a
 
 
 def argsort(a, axis=-1, kind='quicksort', order=None):
@@ -595,6 +826,7 @@
     sort : Describes sorting algorithms used.
     lexsort : Indirect stable sort with multiple keys.
     ndarray.sort : Inplace sort.
+    argpartition : Indirect partial sort.
 
     Notes
     -----
@@ -640,7 +872,11 @@
     array([0, 1])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        argsort = a.argsort
+    except AttributeError:
+        return _wrapit(a, 'argsort', axis, kind, order)
+    return argsort(axis, kind, order)
 
 
 def argmax(a, axis=None):
@@ -694,9 +930,11 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmax'):
-        a = numpyarray(a)
-    return a.argmax()
+    try:
+        argmax = a.argmax
+    except AttributeError:
+        return _wrapit(a, 'argmax')
+    return argmax()
 
 
 def argmin(a, axis=None):
@@ -710,29 +948,37 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmin'):
-        a = numpyarray(a)
-    return a.argmin()
-
-
-def searchsorted(a, v, side='left'):
+    try:
+        argmin = a.argmin
+    except AttributeError:
+        return _wrapit(a, 'argmin')
+    return argmin()
+
+
+def searchsorted(a, v, side='left', sorter=None):
     """
     Find indices where elements should be inserted to maintain order.
 
-    Find the indices into a sorted array `a` such that, if the corresponding
-    elements in `v` were inserted before the indices, the order of `a` would
-    be preserved.
+    Find the indices into a sorted array `a` such that, if the
+    corresponding elements in `v` were inserted before the indices, the
+    order of `a` would be preserved.
 
     Parameters
     ----------
     a : 1-D array_like
-        Input array, sorted in ascending order.
+        Input array. If `sorter` is None, then it must be sorted in
+        ascending order, otherwise `sorter` must be an array of indices
+        that sort it.
     v : array_like
         Values to insert into `a`.
     side : {'left', 'right'}, optional
-        If 'left', the index of the first suitable location found is given.  If
-        'right', return the last such index.  If there is no suitable
+        If 'left', the index of the first suitable location found is given.
+        If 'right', return the last such index.  If there is no suitable
         index, return either 0 or N (where N is the length of `a`).
+    sorter : 1-D array_like, optional
+        .. versionadded:: 1.7.0
+        Optional array of integer indices that sort array a into ascending
+        order. They are typically the result of argsort.
 
     Returns
     -------
@@ -761,7 +1007,11 @@
     array([0, 5, 1, 2])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        searchsorted = a.searchsorted
+    except AttributeError:
+        return _wrapit(a, 'searchsorted', v, side, sorter)
+    return searchsorted(v, side, sorter)
 
 
 def resize(a, new_shape):
@@ -802,10 +1052,30 @@
            [0, 1, 2, 3]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def squeeze(a):
+    if isinstance(new_shape, (int, nt.integer)):
+        new_shape = (new_shape,)
+    a = ravel(a)
+    Na = len(a)
+    if not Na: return mu.zeros(new_shape, a.dtype.char)
+    total_size = um.multiply.reduce(new_shape)
+    n_copies = int(total_size / Na)
+    extra = total_size % Na
+
+    if total_size == 0:
+        return a[:0]
+
+    if extra != 0:
+        n_copies = n_copies+1
+        extra = Na-extra
+
+    a = concatenate( (a,)*n_copies)
+    if extra > 0:
+        a = a[:-extra]
+
+    return reshape(a, new_shape)
+
+
+def squeeze(a, axis=None):
     """
     Remove single-dimensional entries from the shape of an array.
 
@@ -813,12 +1083,19 @@
     ----------
     a : array_like
         Input data.
+    axis : None or int or tuple of ints, optional
+        .. versionadded:: 1.7.0
+
+        Selects a subset of the single-dimensional entries in the
+        shape. If an axis is selected with shape entry greater than
+        one, an error is raised.
 
     Returns
     -------
     squeezed : ndarray
-        The input array, but with with all dimensions of length 1
-        removed.  Whenever possible, a view on `a` is returned.
+        The input array, but with with all or a subset of the
+        dimensions of length 1 removed. This is always `a` itself
+        or a view into `a`.
 
     Examples
     --------
@@ -827,9 +1104,20 @@
     (1, 3, 1)
     >>> np.squeeze(x).shape
     (3,)
+    >>> np.squeeze(x, axis=(2,)).shape
+    (1, 3)
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        squeeze = a.squeeze
+    except AttributeError:
+        return _wrapit(a, 'squeeze')
+    try:
+        # First try to use the new axis= parameter
+        return squeeze(axis=axis)
+    except TypeError:
+        # For backwards compatibility
+        return squeeze()
 
 
 def diagonal(a, offset=0, axis1=0, axis2=1):
@@ -844,6 +1132,27 @@
     removing `axis1` and `axis2` and appending an index to the right equal
     to the size of the resulting diagonals.
 
+    In versions of NumPy prior to 1.7, this function always returned a new,
+    independent array containing a copy of the values in the diagonal.
+
+    In NumPy 1.7 and 1.8, it continues to return a copy of the diagonal,
+    but depending on this fact is deprecated. Writing to the resulting
+    array continues to work as it used to, but a FutureWarning is issued.
+
+    In NumPy 1.9 it returns a read-only view on the original array.
+    Attempting to write to the resulting array will produce an error.
+
+    In NumPy 1.10, it will return a read/write view, Writing to the returned
+    array will alter your original array.
+
+    If you don't write to the array returned by this function, then you can
+    just ignore all of the above.
+
+    If you depend on the current behavior, then we suggest copying the
+    returned array explicitly, i.e., use ``np.diagonal(a).copy()`` instead of
+    just ``np.diagonal(a)``. This will work with both past and future versions
+    of NumPy.
+
     Parameters
     ----------
     a : array_like
@@ -913,7 +1222,7 @@
            [5, 7]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return asarray(a).diagonal(offset, axis1, axis2)
 
 
 def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None):
@@ -972,7 +1281,7 @@
     (2, 3)
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return asarray(a).trace(offset, axis1, axis2, dtype, out)
 
 def ravel(a, order='C'):
     """
@@ -984,21 +1293,25 @@
     Parameters
     ----------
     a : array_like
-        Input array.  The elements in ``a`` are read in the order specified by
+        Input array.  The elements in `a` are read in the order specified by
         `order`, and packed as a 1-D array.
     order : {'C','F', 'A', 'K'}, optional
-        The elements of ``a`` are read in this order. 'C' means to view
-        the elements in C (row-major) order. 'F' means to view the elements
-        in Fortran (column-major) order. 'A' means to view the elements
-        in 'F' order if a is Fortran contiguous, 'C' order otherwise.
-        'K' means to view the elements in the order they occur in memory,
-        except for reversing the data when strides are negative.
-        By default, 'C' order is used.
+        The elements of `a` are read using this index order. 'C' means to
+        index the elements in C-like order, with the last axis index changing
+        fastest, back to the first axis index changing slowest.   'F' means to
+        index the elements in Fortran-like index order, with the first index
+        changing fastest, and the last index changing slowest. Note that the 'C'
+        and 'F' options take no account of the memory layout of the underlying
+        array, and only refer to the order of axis indexing.  'A' means to read
+        the elements in Fortran-like index order if `a` is Fortran *contiguous*
+        in memory, C-like order otherwise.  'K' means to read the elements in
+        the order they occur in memory, except for reversing the data when
+        strides are negative.  By default, 'C' index order is used.
 
     Returns
     -------
     1d_array : ndarray
-        Output of the same dtype as `a`, and of shape ``(a.size(),)``.
+        Output of the same dtype as `a`, and of shape ``(a.size,)``.
 
     See Also
     --------
@@ -1008,11 +1321,11 @@
 
     Notes
     -----
-    In row-major order, the row index varies the slowest, and the column
-    index the quickest.  This can be generalized to multiple dimensions,
-    where row-major order implies that the index along the first axis
-    varies slowest, and the index along the last quickest.  The opposite holds
-    for Fortran-, or column-major, mode.
+    In C-like (row-major) order, in two dimensions, the row index varies the
+    slowest, and the column index the quickest.  This can be generalized to
+    multiple dimensions, where row-major order implies that the index along the
+    first axis varies slowest, and the index along the last quickest.  The
+    opposite holds for Fortran-like, or column-major, index ordering.
 
     Examples
     --------
@@ -1056,9 +1369,8 @@
     array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11])
 
     """
-    if not hasattr(a, 'ravel'):
-        a = numpyarray(a)
-    return a.ravel(order=order)
+    return asarray(a).ravel(order)
+
 
 def nonzero(a):
     """
@@ -1180,9 +1492,11 @@
     (2,)
 
     """
-    if not hasattr(a, 'shape'):
-        a = numpyarray(a)
-    return a.shape
+    try:
+        result = a.shape
+    except AttributeError:
+        result = asarray(a).shape
+    return result
 
 
 def compress(condition, a, axis=None, out=None):
@@ -1217,7 +1531,8 @@
     See Also
     --------
     take, choose, diag, diagonal, select
-    ndarray.compress : Equivalent method.
+    ndarray.compress : Equivalent method in ndarray
+    np.extract: Equivalent method when working on 1-D arrays
     numpy.doc.ufuncs : Section "Output arguments"
 
     Examples
@@ -1244,7 +1559,11 @@
     array([2])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        compress = a.compress
+    except AttributeError:
+        return _wrapit(a, 'compress', condition, axis, out)
+    return compress(condition, axis, out)
 
 
 def clip(a, a_min, a_max, out=None):
@@ -1297,12 +1616,14 @@
     array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8])
 
     """
-    if not hasattr(a, 'clip'):
-        a = numpyarray(a)
-    return a.clip(a_min, a_max, out=out)
-
-
-def sum(a, axis=None, dtype=None, out=None):
+    try:
+        clip = a.clip
+    except AttributeError:
+        return _wrapit(a, 'clip', a_min, a_max, out)
+    return clip(a_min, a_max, out)
+
+
+def sum(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Sum of array elements over a given axis.
 
@@ -1310,9 +1631,16 @@
     ----------
     a : array_like
         Elements to sum.
-    axis : integer, optional
-        Axis over which the sum is taken. By default `axis` is None,
-        and all elements are summed.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a sum is performed.
+        The default (`axis` = `None`) is perform a sum over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a sum is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     dtype : dtype, optional
         The type of the returned array and of the accumulator in which
         the elements are summed.  By default, the dtype of `a` is used.
@@ -1325,6 +1653,10 @@
         (the shape of `a` with `axis` removed, i.e.,
         ``numpy.delete(a.shape, axis)``).  Its type is preserved. See
         `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1368,13 +1700,25 @@
     -128
 
     """
-    assert dtype is None
-    if not hasattr(a, "sum"):
-        a = numpyarray(a)
-    return a.sum(axis=axis, out=out)
-
-
-def product (a, axis=None, dtype=None, out=None):
+    if isinstance(a, _gentype):
+        res = _sum_(a)
+        if out is not None:
+            out[...] = res
+            return out
+        return res
+    elif type(a) is not mu.ndarray:
+        try:
+            sum = a.sum
+        except AttributeError:
+            return _methods._sum(a, axis=axis, dtype=dtype,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameters here...
+        return sum(axis=axis, dtype=dtype, out=out)
+    else:
+        return _methods._sum(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
+
+def product (a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Return the product of array elements over a given axis.
 
@@ -1383,10 +1727,10 @@
     prod : equivalent function; see for details.
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def sometrue(a, axis=None, out=None):
+    return um.multiply.reduce(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+
+def sometrue(a, axis=None, out=None, keepdims=False):
     """
     Check whether some values are true.
 
@@ -1397,14 +1741,14 @@
     any : equivalent function
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'any'):
-        a = numpyarray(a)
-    return a.any()
-
-
-def alltrue (a, axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.any(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.any(axis=axis, out=out)
+
+def alltrue (a, axis=None, out=None, keepdims=False):
     """
     Check if all elements of input array are true.
 
@@ -1413,13 +1757,14 @@
     numpy.all : Equivalent function; see for details.
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'all'):
-        a = numpyarray(a)
-    return a.all()
-
-def any(a,axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.all(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.all(axis=axis, out=out)
+
+def any(a, axis=None, out=None, keepdims=False):
     """
     Test whether any array element along a given axis evaluates to True.
 
@@ -1429,17 +1774,26 @@
     ----------
     a : array_like
         Input array or object that can be converted to an array.
-    axis : int, optional
-        Axis along which a logical OR is performed.  The default
-        (`axis` = `None`) is to perform a logical OR over a flattened
-        input array. `axis` may be negative, in which case it counts
-        from the last to the first axis.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical OR reduction is performed.
+        The default (`axis` = `None`) is perform a logical OR over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a reduction is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     out : ndarray, optional
         Alternate output array in which to place the result.  It must have
         the same shape as the expected output and its type is preserved
         (e.g., if it is of type float, then it will remain so, returning
         1.0 for True and 0.0 for False, regardless of the type of `a`).
         See `doc.ufuncs` (Section "Output arguments") for details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1483,14 +1837,14 @@
     (191614240, 191614240)
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'any'):
-        a = numpyarray(a)
-    return a.any()
-
-
-def all(a,axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.any(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.any(axis=axis, out=out)
+
+def all(a, axis=None, out=None, keepdims=False):
     """
     Test whether all array elements along a given axis evaluate to True.
 
@@ -1498,17 +1852,26 @@
     ----------
     a : array_like
         Input array or object that can be converted to an array.
-    axis : int, optional
-        Axis along which a logical AND is performed.
-        The default (`axis` = `None`) is to perform a logical AND
-        over a flattened input array.  `axis` may be negative, in which
-        case it counts from the last to the first axis.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical AND reduction is performed.
+        The default (`axis` = `None`) is perform a logical OR over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a reduction is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     out : ndarray, optional
         Alternate output array in which to place the result.
         It must have the same shape as the expected output and its
         type is preserved (e.g., if ``dtype(out)`` is float, the result
         will consist of 0.0's and 1.0's).  See `doc.ufuncs` (Section
         "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1547,12 +1910,12 @@
     (28293632, 28293632, array([ True], dtype=bool))
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'all'):
-        a = numpyarray(a)
-    return a.all()
-
+    arr = asanyarray(a)
+
+    try:
+        return arr.all(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.all(axis=axis, out=out)
 
 def cumsum (a, axis=None, dtype=None, out=None):
     """
@@ -1592,6 +1955,8 @@
 
     trapz : Integration of array values using the composite trapezoidal rule.
 
+    diff :  Calculate the n-th order discrete difference along given axis.
+
     Notes
     -----
     Arithmetic is modular when using integer types, and no error is
@@ -1616,7 +1981,11 @@
            [ 4,  9, 15]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumsum = a.cumsum
+    except AttributeError:
+        return _wrapit(a, 'cumsum', axis, dtype, out)
+    return cumsum(axis, dtype, out)
 
 
 def cumproduct(a, axis=None, dtype=None, out=None):
@@ -1629,7 +1998,11 @@
     cumprod : equivalent function; see for details.
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumprod = a.cumprod
+    except AttributeError:
+        return _wrapit(a, 'cumprod', axis, dtype, out)
+    return cumprod(axis, dtype, out)
 
 
 def ptp(a, axis=None, out=None):
@@ -1670,10 +2043,14 @@
     array([1, 1])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def amax(a, axis=None, out=None):
+    try:
+        ptp = a.ptp
+    except AttributeError:
+        return _wrapit(a, 'ptp', axis, out)
+    return ptp(axis, out)
+
+
+def amax(a, axis=None, out=None, keepdims=False):
     """
     Return the maximum of an array or maximum along an axis.
 
@@ -1682,11 +2059,15 @@
     a : array_like
         Input data.
     axis : int, optional
-        Axis along which to operate.  By default flattened input is used.
+        Axis along which to operate.  By default, flattened input is used.
     out : ndarray, optional
-        Alternate output array in which to place the result.  Must be of
-        the same shape and buffer length as the expected output.  See
-        `doc.ufuncs` (Section "Output arguments") for more details.
+        Alternative output array in which to place the result.  Must
+        be of the same shape and buffer length as the expected output.
+        See `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1697,27 +2078,40 @@
 
     See Also
     --------
-    nanmax : NaN values are ignored instead of being propagated.
-    fmax : same behavior as the C99 fmax function.
-    argmax : indices of the maximum values.
+    amin :
+        The minimum value of an array along a given axis, propagating any NaNs.
+    nanmax :
+        The maximum value of an array along a given axis, ignoring any NaNs.
+    maximum :
+        Element-wise maximum of two arrays, propagating any NaNs.
+    fmax :
+        Element-wise maximum of two arrays, ignoring any NaNs.
+    argmax :
+        Return the indices of the maximum values.
+
+    nanmin, minimum, fmin
 
     Notes
     -----
     NaN values are propagated, that is if at least one item is NaN, the
-    corresponding max value will be NaN as well.  To ignore NaN values
+    corresponding max value will be NaN as well. To ignore NaN values
     (MATLAB behavior), please use nanmax.
 
+    Don't use `amax` for element-wise comparison of 2 arrays; when
+    ``a.shape[0]`` is 2, ``maximum(a[0], a[1])`` is faster than
+    ``amax(a, axis=0)``.
+
     Examples
     --------
     >>> a = np.arange(4).reshape((2,2))
     >>> a
     array([[0, 1],
            [2, 3]])
-    >>> np.amax(a)
+    >>> np.amax(a)           # Maximum of the flattened array
     3
-    >>> np.amax(a, axis=0)
+    >>> np.amax(a, axis=0)   # Maxima along the first axis
     array([2, 3])
-    >>> np.amax(a, axis=1)
+    >>> np.amax(a, axis=1)   # Maxima along the second axis
     array([1, 3])
 
     >>> b = np.arange(5, dtype=np.float)
@@ -1728,14 +2122,19 @@
     4.0
 
     """
-    if not hasattr(a, "max"):
-        a = numpyarray(a)
-    if a.size < 1:
-        return numpyarray([])
-    return a.max(axis=axis, out=out)
-
-
-def amin(a, axis=None, out=None):
+    if type(a) is not mu.ndarray:
+        try:
+            amax = a.max
+        except AttributeError:
+            return _methods._amax(a, axis=axis,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameter
+        return amax(axis=axis, out=out)
+    else:
+        return _methods._amax(a, axis=axis,
+                            out=out, keepdims=keepdims)
+
+def amin(a, axis=None, out=None, keepdims=False):
     """
     Return the minimum of an array or minimum along an axis.
 
@@ -1744,30 +2143,47 @@
     a : array_like
         Input data.
     axis : int, optional
-        Axis along which to operate.  By default a flattened input is used.
+        Axis along which to operate.  By default, flattened input is used.
     out : ndarray, optional
         Alternative output array in which to place the result.  Must
         be of the same shape and buffer length as the expected output.
         See `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
-    amin : ndarray
-        A new array or a scalar array with the result.
+    amin : ndarray or scalar
+        Minimum of `a`. If `axis` is None, the result is a scalar value.
+        If `axis` is given, the result is an array of dimension
+        ``a.ndim - 1``.
 
     See Also
     --------
-    nanmin: nan values are ignored instead of being propagated
-    fmin: same behavior as the C99 fmin function
-    argmin: Return the indices of the minimum values.
-
-    amax, nanmax, fmax
+    amax :
+        The maximum value of an array along a given axis, propagating any NaNs.
+    nanmin :
+        The minimum value of an array along a given axis, ignoring any NaNs.
+    minimum :
+        Element-wise minimum of two arrays, propagating any NaNs.
+    fmin :
+        Element-wise minimum of two arrays, ignoring any NaNs.
+    argmin :
+        Return the indices of the minimum values.
+
+    nanmax, maximum, fmax
 
     Notes
     -----
-    NaN values are propagated, that is if at least one item is nan, the
-    corresponding min value will be nan as well. To ignore NaN values (matlab
-    behavior), please use nanmin.
+    NaN values are propagated, that is if at least one item is NaN, the
+    corresponding min value will be NaN as well. To ignore NaN values
+    (MATLAB behavior), please use nanmin.
+
+    Don't use `amin` for element-wise comparison of 2 arrays; when
+    ``a.shape[0]`` is 2, ``minimum(a[0], a[1])`` is faster than
+    ``amin(a, axis=0)``.
 
     Examples
     --------
@@ -1777,9 +2193,9 @@
            [2, 3]])
     >>> np.amin(a)           # Minimum of the flattened array
     0
-    >>> np.amin(a, axis=0)         # Minima along the first axis
+    >>> np.amin(a, axis=0)   # Minima along the first axis
     array([0, 1])
-    >>> np.amin(a, axis=1)         # Minima along the second axis
+    >>> np.amin(a, axis=1)   # Minima along the second axis
     array([0, 2])
 
     >>> b = np.arange(5, dtype=np.float)
@@ -1790,11 +2206,17 @@
     0.0
 
     """
-    if not hasattr(a, 'min'):
-        a = numpyarray(a)
-    if a.size < 1:
-        return numpyarray([])
-    return a.min(axis=axis, out=out)
+    if type(a) is not mu.ndarray:
+        try:
+            amin = a.min
+        except AttributeError:
+            return _methods._amin(a, axis=axis,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameter
+        return amin(axis=axis, out=out)
+    else:
+        return _methods._amin(a, axis=axis,
+                            out=out, keepdims=keepdims)
 
 def alen(a):
     """
@@ -1807,7 +2229,7 @@
 
     Returns
     -------
-    l : int
+    alen : int
        Length of the first dimension of `a`.
 
     See Also
@@ -1823,12 +2245,13 @@
     7
 
     """
-    if not hasattr(a, 'shape'):
-        a = numpyarray(a)
-    return a.shape[0]
-
-
-def prod(a, axis=None, dtype=None, out=None):
+    try:
+        return len(a)
+    except TypeError:
+        return len(array(a, ndmin=1))
+
+
+def prod(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Return the product of array elements over a given axis.
 
@@ -1836,9 +2259,16 @@
     ----------
     a : array_like
         Input data.
-    axis : int, optional
-        Axis over which the product is taken.  By default, the product
-        of all elements is calculated.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a product is performed.
+        The default (`axis` = `None`) is perform a product over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a product is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     dtype : data-type, optional
         The data-type of the returned array, as well as of the accumulator
         in which the elements are multiplied.  By default, if `a` is of
@@ -1849,6 +2279,10 @@
         Alternative output array in which to place the result. It must have
         the same shape as the expected output, but the type of the
         output values will be cast if necessary.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1902,8 +2336,16 @@
     True
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
+    if type(a) is not mu.ndarray:
+        try:
+            prod = a.prod
+        except AttributeError:
+            return _methods._prod(a, axis=axis, dtype=dtype,
+                                out=out, keepdims=keepdims)
+        return prod(axis=axis, dtype=dtype, out=out)
+    else:
+        return _methods._prod(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
 
 def cumprod(a, axis=None, dtype=None, out=None):
     """
@@ -1965,7 +2407,11 @@
            [  4,  20, 120]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumprod = a.cumprod
+    except AttributeError:
+        return _wrapit(a, 'cumprod', axis, dtype, out)
+    return cumprod(axis, dtype, out)
 
 
 def ndim(a):
@@ -1999,9 +2445,10 @@
     0
 
     """
-    if not hasattr(a, 'ndim'):
-        a = numpyarray(a)
-    return a.ndim
+    try:
+        return a.ndim
+    except AttributeError:
+        return asarray(a).ndim
 
 
 def rank(a):
@@ -2044,9 +2491,10 @@
     0
 
     """
-    if not hasattr(a, 'ndim'):
-        a = numpyarray(a)
-    return a.ndim
+    try:
+        return a.ndim
+    except AttributeError:
+        return asarray(a).ndim
 
 
 def size(a, axis=None):
@@ -2083,7 +2531,16 @@
     2
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    if axis is None:
+        try:
+            return a.size
+        except AttributeError:
+            return asarray(a).size
+    else:
+        try:
+            return a.shape[axis]
+        except AttributeError:
+            return asarray(a).shape[axis]
 
 
 def around(a, decimals=0, out=None):
@@ -2152,7 +2609,11 @@
     array([ 0,  0,  0, 10])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        round = a.round
+    except AttributeError:
+        return _wrapit(a, 'round', decimals, out)
+    return round(decimals, out)
 
 
 def round_(a, decimals=0, out=None):
@@ -2166,10 +2627,14 @@
     around : equivalent function
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def mean(a, axis=None, dtype=None, out=None):
+    try:
+        round = a.round
+    except AttributeError:
+        return _wrapit(a, 'round', decimals, out)
+    return round(decimals, out)
+
+
+def mean(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Compute the arithmetic mean along the specified axis.
 
@@ -2194,6 +2659,10 @@
         is ``None``; if provided, it must have the same shape as the
         expected output, but the type will be cast if necessary.
         See `doc.ufuncs` for details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2204,6 +2673,7 @@
     See Also
     --------
     average : Weighted average
+    std, var, nanmean, nanstd, nanvar
 
     Notes
     -----
@@ -2240,14 +2710,17 @@
     0.55000000074505806
 
     """
-    assert dtype is None
-    assert out is None
-    if not hasattr(a, "mean"):
-        a = numpyarray(a)
-    return a.mean(axis=axis)
-
-
-def std(a, axis=None, dtype=None, out=None, ddof=0):
+    if type(a) is not mu.ndarray:
+        try:
+            mean = a.mean
+            return mean(axis=axis, dtype=dtype, out=out)
+        except AttributeError:
+            pass
+
+    return _methods._mean(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
+
+def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
     """
     Compute the standard deviation along the specified axis.
 
@@ -2274,6 +2747,10 @@
         Means Delta Degrees of Freedom.  The divisor used in calculations
         is ``N - ddof``, where ``N`` represents the number of elements.
         By default `ddof` is zero.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2283,7 +2760,7 @@
 
     See Also
     --------
-    var, mean
+    var, mean, nanmean, nanstd, nanvar
     numpy.doc.ufuncs : Section "Output arguments"
 
     Notes
@@ -2291,14 +2768,15 @@
     The standard deviation is the square root of the average of the squared
     deviations from the mean, i.e., ``std = sqrt(mean(abs(x - x.mean())**2))``.
 
-    The average squared deviation is normally calculated as ``x.sum() / N``, where
-    ``N = len(x)``.  If, however, `ddof` is specified, the divisor ``N - ddof``
-    is used instead. In standard statistical practice, ``ddof=1`` provides an
-    unbiased estimator of the variance of the infinite population. ``ddof=0``
-    provides a maximum likelihood estimate of the variance for normally
-    distributed variables. The standard deviation computed in this function
-    is the square root of the estimated variance, so even with ``ddof=1``, it
-    will not be an unbiased estimate of the standard deviation per se.
+    The average squared deviation is normally calculated as
+    ``x.sum() / N``, where ``N = len(x)``.  If, however, `ddof` is specified,
+    the divisor ``N - ddof`` is used instead. In standard statistical
+    practice, ``ddof=1`` provides an unbiased estimator of the variance
+    of the infinite population. ``ddof=0`` provides a maximum likelihood
+    estimate of the variance for normally distributed variables. The
+    standard deviation computed in this function is the square root of
+    the estimated variance, so even with ``ddof=1``, it will not be an
+    unbiased estimate of the standard deviation per se.
 
     Note that, for complex numbers, `std` takes the absolute
     value before squaring, so that the result is always real and nonnegative.
@@ -2333,15 +2811,18 @@
     0.44999999925552653
 
     """
-    assert dtype is None
-    assert out is None
-    assert ddof == 0
-    if not hasattr(a, "std"):
-        a = numpyarray(a)
-    return a.std(axis=axis)
-
-
-def var(a, axis=None, dtype=None, out=None, ddof=0):
+    if type(a) is not mu.ndarray:
+        try:
+            std = a.std
+            return std(axis=axis, dtype=dtype, out=out, ddof=ddof)
+        except AttributeError:
+            pass
+
+    return _methods._std(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
+                                keepdims=keepdims)
+
+def var(a, axis=None, dtype=None, out=None, ddof=0,
+                            keepdims=False):
     """
     Compute the variance along the specified axis.
 
@@ -2369,6 +2850,10 @@
         "Delta Degrees of Freedom": the divisor used in the calculation is
         ``N - ddof``, where ``N`` represents the number of elements. By
         default `ddof` is zero.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2378,8 +2863,7 @@
 
     See Also
     --------
-    std : Standard deviation
-    mean : Average
+    std , mean, nanmean, nanstd, nanvar
     numpy.doc.ufuncs : Section "Output arguments"
 
     Notes
@@ -2408,9 +2892,9 @@
     >>> a = np.array([[1,2],[3,4]])
     >>> np.var(a)
     1.25
-    >>> np.var(a,0)
+    >>> np.var(a, axis=0)
     array([ 1.,  1.])
-    >>> np.var(a,1)
+    >>> np.var(a, axis=1)
     array([ 0.25,  0.25])
 
     In single precision, var() can be inaccurate:
@@ -2421,7 +2905,7 @@
     >>> np.var(a)
     0.20405951142311096
 
-    Computing the standard deviation in float64 is more accurate:
+    Computing the variance in float64 is more accurate:
 
     >>> np.var(a, dtype=np.float64)
     0.20249999932997387
@@ -2429,10 +2913,12 @@
     0.20250000000000001
 
     """
-    assert dtype is None
-    assert out is None
-    assert ddof == 0
-    if not hasattr(a, "var"):
-        a = numpyarray(a)
-    return a.var(axis=axis)
-
+    if type(a) is not mu.ndarray:
+        try:
+            var = a.var
+            return var(axis=axis, dtype=dtype, out=out, ddof=ddof)
+        except AttributeError:
+            pass
+
+    return _methods._var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
+                                keepdims=keepdims)
diff --git a/lib_pypy/numpypy/core/shape_base.py b/lib_pypy/numpypy/core/shape_base.py
--- a/lib_pypy/numpypy/core/shape_base.py
+++ b/lib_pypy/numpypy/core/shape_base.py
@@ -1,7 +1,9 @@
+from __future__ import division, absolute_import, print_function
+
 __all__ = ['atleast_1d', 'atleast_2d', 'atleast_3d', 'vstack', 'hstack']
 
-import numeric as _nx
-from numeric import array, asanyarray, newaxis
+from . import numeric as _nx
+from .numeric import array, asanyarray, newaxis
 
 def atleast_1d(*arys):
     """
@@ -55,7 +57,6 @@
     else:
         return res
 
-
 def atleast_2d(*arys):
     """
     View inputs as arrays with at least two dimensions.
@@ -99,7 +100,7 @@
         if len(ary.shape) == 0 :
             result = ary.reshape(1, 1)
         elif len(ary.shape) == 1 :
-            result = ary[newaxis, :]
+            result = ary[newaxis,:]
         else :
             result = ary
         res.append(result)
@@ -161,11 +162,11 @@
     for ary in arys:
         ary = asanyarray(ary)
         if len(ary.shape) == 0:
-            result = ary.reshape(1,1,1)
+            result = ary.reshape(1, 1, 1)
         elif len(ary.shape) == 1:
-            result = ary[newaxis,:,newaxis]
+            result = ary[newaxis,:, newaxis]
         elif len(ary.shape) == 2:
-            result = ary[:,:,newaxis]
+            result = ary[:,:, newaxis]
         else:
             result = ary
         res.append(result)
@@ -174,6 +175,7 @@
     else:
         return res
 
+
 def vstack(tup):
     """
     Stack arrays in sequence vertically (row wise).
@@ -223,7 +225,7 @@
            [4]])
 
     """
-    return _nx.concatenate(map(atleast_2d,tup),0)
+    return _nx.concatenate([atleast_2d(_m) for _m in tup], 0)
 
 def hstack(tup):
     """
@@ -267,7 +269,7 @@
            [3, 4]])
 
     """
-    arrs = map(atleast_1d,tup)
+    arrs = [atleast_1d(_m) for _m in tup]
     # As a special case, dimension 0 of 1-dimensional arrays is "horizontal"
     if arrs[0].ndim == 1:
         return _nx.concatenate(arrs, 0)

From noreply at buildbot.pypy.org  Tue Oct 15 09:14:45 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 09:14:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: add array_equiv
Message-ID: <20131015071445.7510F1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67383:075052ce29b7
Date: 2013-10-15 03:11 -0400
http://bitbucket.org/pypy/pypy/changeset/075052ce29b7/

Log:	add array_equiv

diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -4,7 +4,7 @@
            'newaxis', 'ufunc',
            'asarray', 'asanyarray', 'base_repr',
            'array_repr', 'array_str', 'set_string_function',
-           'array_equal', 'outer', 'vdot', 'identity', 'little_endian',
+           'array_equal', 'array_equiv', 'outer', 'vdot', 'identity', 'little_endian',
            'Inf', 'inf', 'infty', 'Infinity', 'nan', 'NaN', 'False_', 'True_',
            'seterr',
           ]
@@ -453,6 +453,50 @@
         return False
     return bool((a1 == a2).all())
 
+def array_equiv(a1, a2):
+    """
+    Returns True if input arrays are shape consistent and all elements equal.
+
+    Shape consistent means they are either the same shape, or one input array
+    can be broadcasted to create the same shape as the other one.
+
+    Parameters
+    ----------
+    a1, a2 : array_like
+        Input arrays.
+
+    Returns
+    -------
+    out : bool
+        True if equivalent, False otherwise.
+
+    Examples
+    --------
+    >>> np.array_equiv([1, 2], [1, 2])
+    True
+    >>> np.array_equiv([1, 2], [1, 3])
+    False
+
+    Showing the shape equivalence:
+
+    >>> np.array_equiv([1, 2], [[1, 2], [1, 2]])
+    True
+    >>> np.array_equiv([1, 2], [[1, 2, 1, 2], [1, 2, 1, 2]])
+    False
+
+    >>> np.array_equiv([1, 2], [[1, 2], [1, 3]])
+    False
+
+    """
+    try:
+        a1, a2 = asarray(a1), asarray(a2)
+    except:
+        return False
+    try:
+        return bool(asarray(a1 == a2).all())
+    except ValueError:
+        return False
+
 def outer(a,b):
     """
     Compute the outer product of two vectors.
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
@@ -177,6 +177,15 @@
         assert not array_equal(array(a), b)
         assert not array_equal(array(a), array(b))
 
+    def test_equiv(self):
+        import numpypy as np
+
+        assert np.array_equiv([1, 2], [1, 2])
+        assert not np.array_equiv([1, 2], [1, 3])
+        assert np.array_equiv([1, 2], [[1, 2], [1, 2]])
+        assert not np.array_equiv([1, 2], [[1, 2, 1, 2], [1, 2, 1, 2]])
+        assert not np.array_equiv([1, 2], [[1, 2], [1, 3]])
+
 
 class AppTestNumeric(BaseNumpyAppTest):
     def test_outer(self):

From noreply at buildbot.pypy.org  Tue Oct 15 10:20:35 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 15 Oct 2013 10:20:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: add pretty formatting
Message-ID: <20131015082035.05A2C1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67384:6eea90d83440
Date: 2013-10-15 10:19 +0200
http://bitbucket.org/pypy/pypy/changeset/6eea90d83440/

Log:	add pretty formatting

diff --git a/rpython/tool/gcanalyze.py b/rpython/tool/gcanalyze.py
--- a/rpython/tool/gcanalyze.py
+++ b/rpython/tool/gcanalyze.py
@@ -10,7 +10,7 @@
 
 NO_BUCKETS = 8
 
-def main(arg):
+def main(arg, plot=False):
     log = parse_log(open(arg).readlines())
     all = []
     for entry in log:
@@ -18,12 +18,13 @@
             start = entry[1]
             end = entry[2]
             all.append(float(end - start) / 1000000)
-    format_output(all)
+    format_output(all, plot=plot)
 
-def format_output(all):
+def format_output(all, plot=False):
     avg = sum(all) / len(all)
     max_t = max(all)
     print "AVG:", "%.1fms" % avg, "MAX:", "%.1fms" % max_t, "TOTAL:" , "%.1fms" % sum(all)
+    print
     buckets = [0] * NO_BUCKETS
     for item in all:
         bucket = int(item / max_t * NO_BUCKETS)
@@ -34,8 +35,20 @@
     l2 = [str(i) for i in buckets]
     for i, elem in enumerate(l1):
         l2[i] += " " * (len(elem) - len(l2[i]))
-    print " ".join(l1)
-    print " ".join(l2)
+    if plot:
+        l3 = ["+"]
+        for i, elem in enumerate(l1):
+            l3.append("-" * len(elem) + "+")
+        l1.insert(0, "")
+        l2.insert(0, "")
+        print "".join(l3)
+        print "|".join(l1) + "|"
+        print "".join(l3)
+        print "|".join(l2) + "|"
+        print "".join(l3)
+    else:
+        print " ".join(l1)
+        print " ".join(l2)
 
 if __name__ == '__main__':
     if len(sys.argv) < 2 or len(sys.argv) > 3:
@@ -54,4 +67,4 @@
             sys.exit(1)
     else:
         arg = sys.argv[1]
-    main(arg)
+    main(arg, plot=plot)

From noreply at buildbot.pypy.org  Tue Oct 15 10:21:24 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 15 Oct 2013 10:21:24 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: work on the benchmark part,
 add an XXX and a hilarious typo
Message-ID: <20131015082124.D9A0C1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5079:b20a0efba9e5
Date: 2013-10-15 10:21 +0200
http://bitbucket.org/pypy/extradoc/changeset/b20a0efba9e5/

Log:	work on the benchmark part, add an XXX and a hilarious typo

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -44,11 +44,83 @@
 Benchmarks
 ==========
 
+Everyone loves benchmarks. We did not measure any significant speed difference
+on our quite extensive benchmark suite on speed.pypy.org. The main
+benchmark that we used for other comparisons was translating `topaz`_
+ruby interpreter using various versions of PyPy and CPython. The exact
+command was ``python <pypy-checkout>/bin/rpython -O2 --rtype targettopaz.py``.
+Versions:
 
+* topaz - dce3eef7b1910fc5600a4cd0afd6220543104823
+* pypy source - defb5119e3c6
+* pypy compiled with minimark (non-incremental GC) - d1a0c07b6586
+* pypy compiled with incminimark (new, incremental GC) - 417a7117f8d7
+* CPython - 2.7.3
+
+The memory usage of CPython, PyPy with minimark and PyPy with incminimark is
+shown here. Note that this benchmark is quite bad for PyPy in general, the
+memory usage is higher and the amount of time taken is longer. This is due
+to JIT warmup being both memory hungry and inefficient. We'll work on it next.
+But first, the new GC is not worst than the old one.
+
+.. image:: memusage.png
+
+The image was obtained by graphing the output of `memusage.py`_.
+
+.. _`topaz`: http://http://docs.topazruby.com/en/latest/
+.. _`memusage.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/memusage/memusage.py?at=default
+
+However, the GC pauses are significantly smaller. For PyPy the way to
+get GC pauses is to measure time between start and stop while running stuff
+with ``PYPYLOG=gc-collect:log pypy program.py``, for CPython, the magic
+incantation is ``gc.set_debug(gc.DEBUG_STATS)`` and parsing the output.
+For what is worth, the average and total for CPython, as well as the total
+number of events are not directly comparable since it only shows the cyclic
+collector, not the reference counts. The only comparable thing is the
+amount of long pauses and their duration. In the table below, pause duration
+is sorted into 8 buckets, each meaning "below or equal the threshold".
+The output is generated using `gcanalyze`_ tool.
+
+.. _`gcanalyze`: https://bitbucket.org/pypy/pypy/src/default/rpython/tool/gcanalyze.py?at=default
+
+CPython:
+
++-------+-------+-------+-------+-------+-------+--------+--------+
+|150.1ms|300.2ms|450.3ms|600.5ms|750.6ms|900.7ms|1050.8ms|1200.9ms|
++-------+-------+-------+-------+-------+-------+--------+--------+
+|5417   |5      |3      |2      |1      |1      |0       |1       |
++-------+-------+-------+-------+-------+-------+--------+--------+
+
+
+PyPy minimark (non-incremental GC):
+
++-------+-------+-------+-------+--------+--------+--------+--------+
+|216.4ms|432.8ms|649.2ms|865.6ms|1082.0ms|1298.4ms|1514.8ms|1731.2ms|
++-------+-------+-------+-------+--------+--------+--------+--------+
+|27     |14     |6      |4      |6       |5       |3       |3       |
++-------+-------+-------+-------+--------+--------+--------+--------+
+
+PyPy incminimark (new incremental GC):
+
++------+------+------+------+------+------+-------+-------+
+|15.7ms|31.4ms|47.1ms|62.8ms|78.6ms|94.3ms|110.0ms|125.7ms|
++------+------+------+------+------+------+-------+-------+
+|25512 |122   |4     |1     |0     |0     |0      |2      |
++------+------+------+------+------+------+-------+-------+
+
+As we can see, while there is still work to be done (the 100ms ones could
+be split among several steps), we did improve the situation quite drastically
+without any actual performance difference.
+
+Note about the benchmark - we know it's a pretty extreme case of a JIT
+warmup, we know we suck on it, we're working on it and we're not afraid of
+showing PyPy is not always the best ;-)
 
 Nitty gritty details
 ====================
 
+XXX insert some links where you can read about terms used
+
 This was done as a patch to "minimark", our current GC, and called
 "incminimark" for now.  The former is a generational stop-the-world GC.
 New objects are allocated "young", i.e. in the nursery, a special zone
@@ -59,7 +131,7 @@
 From time to time, this minor collection is followed by a "major
 collection": in that step, we walk *all* objects to classify which ones
 are still alive and which ones are now dead (*marking*) and free the
-memory occupied by the dead ones (*speeding*).
+memory occupied by the dead ones (*sweeping*).
 
 This "major collection" is what gives the long GC pauses.  To fix this
 problem we made the GC incremental: instead of running one complete

From noreply at buildbot.pypy.org  Tue Oct 15 10:32:43 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 15 Oct 2013 10:32:43 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: a clarification
Message-ID: <20131015083243.1BDE31C01B0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5080:94b857d4e394
Date: 2013-10-15 10:31 +0200
http://bitbucket.org/pypy/extradoc/changeset/94b857d4e394/

Log:	a clarification

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -165,7 +165,7 @@
 The trick we used in PyPy is to consider minor collections as part of
 the whole, rather than focus only on major collections.  The existing
 minimark GC had always used a "write barrier" (a piece of code run every time
-you set or get from an object or array) to do its job, like any
+you set or get a pointer from an object or array) to do its job, like any
 generational GC.  This write barrier is used to detect when an old
 object (outside the nursery) is modified to point to a young object
 (inside the nursery), which is essential information for minor

From noreply at buildbot.pypy.org  Tue Oct 15 11:04:07 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 11:04:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: enable HALF typeinfo
Message-ID: <20131015090407.E13B01C1380@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67386:18fc7ef8ae01
Date: 2013-10-15 04:55 -0400
http://bitbucket.org/pypy/pypy/changeset/18fc7ef8ae01/

Log:	enable HALF typeinfo

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -822,7 +822,7 @@
             'UINT': self.w_uint32dtype,
             'INTP': self.w_intpdtype,
             'UINTP': self.w_uintpdtype,
-            #'HALF',
+            'HALF': self.w_float16dtype,
             'BYTE': self.w_int8dtype,
             #'TIMEDELTA',
             'INT': self.w_int32dtype,
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -903,7 +903,7 @@
         BaseNumpyAppTest.setup_class.im_func(cls)
 
     def test_typeinfo(self):
-        from numpypy import void, number, int64, bool_, complex64, complex128
+        from numpypy import void, number, int64, bool_, complex64, complex128, float16
         from numpypy.core.multiarray import typeinfo
         assert typeinfo['Number'] == number
         assert typeinfo['LONGLONG'] == ('q', 9, 64, 8, 9223372036854775807L, -9223372036854775808L, int64)
@@ -911,6 +911,7 @@
         assert typeinfo['BOOL'] == ('?', 0, 8, 1, 1, 0, bool_)
         assert typeinfo['CFLOAT'] == ('F', 14, 64, 4, complex64)
         assert typeinfo['CDOUBLE'] == ('D', 15, 128, 8, complex128)
+        assert typeinfo['HALF'] == ('e', 23, 16, 2, float16)
 
 class AppTestNoLongDoubleDtypes(BaseNumpyAppTest):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Tue Oct 15 11:04:06 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 11:04:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: test/fix CFLOAT/CDOUBLE typeinfo
Message-ID: <20131015090406.A593D1C135D@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67385:0b3416c0ca57
Date: 2013-10-15 04:33 -0400
http://bitbucket.org/pypy/pypy/changeset/0b3416c0ca57/

Log:	test/fix CFLOAT/CDOUBLE typeinfo

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -816,14 +816,14 @@
             #'OBJECT',
             'ULONGLONG': self.w_uint64dtype,
             'STRING': self.w_stringdtype,
-            'CDOUBLE': self.w_complex64dtype,
+            'CFLOAT': self.w_complex64dtype,
+            'CDOUBLE': self.w_complex128dtype,
             #'DATETIME',
             'UINT': self.w_uint32dtype,
             'INTP': self.w_intpdtype,
             'UINTP': self.w_uintpdtype,
             #'HALF',
             'BYTE': self.w_int8dtype,
-            #'CFLOAT': ,
             #'TIMEDELTA',
             'INT': self.w_int32dtype,
             'DOUBLE': self.w_float64dtype,
@@ -856,7 +856,7 @@
                        space.wrap(dtype.num),
                        space.wrap(itemsize * 8), # in case of changing
                        # number of bits per byte in the future
-                       space.wrap(itemsize or 1)]
+                       space.wrap(itemsize / (2 if dtype.kind == COMPLEXLTR else 1) or 1)]
             if dtype.is_int_type():
                 if dtype.kind == BOOLLTR:
                     w_maxobj = space.wrap(1)
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -903,12 +903,14 @@
         BaseNumpyAppTest.setup_class.im_func(cls)
 
     def test_typeinfo(self):
-        from numpypy import void, number, int64, bool_
+        from numpypy import void, number, int64, bool_, complex64, complex128
         from numpypy.core.multiarray import typeinfo
         assert typeinfo['Number'] == number
         assert typeinfo['LONGLONG'] == ('q', 9, 64, 8, 9223372036854775807L, -9223372036854775808L, int64)
         assert typeinfo['VOID'] == ('V', 20, 0, 1, void)
         assert typeinfo['BOOL'] == ('?', 0, 8, 1, 1, 0, bool_)
+        assert typeinfo['CFLOAT'] == ('F', 14, 64, 4, complex64)
+        assert typeinfo['CDOUBLE'] == ('D', 15, 128, 8, complex128)
 
 class AppTestNoLongDoubleDtypes(BaseNumpyAppTest):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Tue Oct 15 11:04:09 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 11:04:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: use numerictypes from numpy
Message-ID: <20131015090409.1DD101D22C2@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67387:cb57a653677d
Date: 2013-10-15 04:59 -0400
http://bitbucket.org/pypy/pypy/changeset/cb57a653677d/

Log:	use numerictypes from numpy

diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -1,6 +1,664 @@
-from _numpypy.numerictypes import *
+######################################################################
+# This is a copy of numpy/core/numerictypes.py modified for numpypy
+######################################################################
+"""
+numerictypes: Define the numeric type objects
 
-from .multiarray import dtype
+This module is designed so "from numerictypes import \\*" is safe.
+Exported symbols include:
+
+  Dictionary with all registered number types (including aliases):
+    typeDict
+
+  Type objects (not all will be available, depends on platform):
+      see variable sctypes for which ones you have
+
+    Bit-width names
+
+    int8 int16 int32 int64 int128
+    uint8 uint16 uint32 uint64 uint128
+    float16 float32 float64 float96 float128 float256
+    complex32 complex64 complex128 complex192 complex256 complex512
+    datetime64 timedelta64
+
+    c-based names
+
+    bool_
+
+    object_
+
+    void, str_, unicode_
+
+    byte, ubyte,
+    short, ushort
+    intc, uintc,
+    intp, uintp,
+    int_, uint,
+    longlong, ulonglong,
+
+    single, csingle,
+    float_, complex_,
+    longfloat, clongfloat,
+
+   As part of the type-hierarchy:    xx -- is bit-width
+
+   generic
+     +-> bool_                                  (kind=b)
+     +-> number                                 (kind=i)
+     |     integer
+     |     signedinteger   (intxx)
+     |     byte
+     |     short
+     |     intc
+     |     intp           int0
+     |     int_
+     |     longlong
+     +-> unsignedinteger  (uintxx)              (kind=u)
+     |     ubyte
+     |     ushort
+     |     uintc
+     |     uintp          uint0
+     |     uint_
+     |     ulonglong
+     +-> inexact
+     |   +-> floating           (floatxx)       (kind=f)
+     |   |     half
+     |   |     single
+     |   |     float_  (double)
+     |   |     longfloat
+     |   \\-> complexfloating    (complexxx)     (kind=c)
+     |         csingle  (singlecomplex)
+     |         complex_ (cfloat, cdouble)
+     |         clongfloat (longcomplex)
+     +-> flexible
+     |     character
+     |     void                                 (kind=V)
+     |
+     |     str_     (string_, bytes_)           (kind=S)    [Python 2]
+     |     unicode_                             (kind=U)    [Python 2]
+     |
+     |     bytes_   (string_)                   (kind=S)    [Python 3]
+     |     str_     (unicode_)                  (kind=U)    [Python 3]
+     |
+     \\-> object_ (not used much)                (kind=O)
+
+"""
+from __future__ import division, absolute_import, print_function
+
+# we add more at the bottom
+__all__ = ['sctypeDict', 'sctypeNA', 'typeDict', 'typeNA', 'sctypes',
+           'ScalarType', 'obj2sctype', 'cast', 'nbytes', 'sctype2char',
+           'maximum_sctype', 'issctype', 'typecodes', 'find_common_type',
+           'issubdtype',
+           ]
+
+from .multiarray import (
+        typeinfo, ndarray, array, empty, dtype,
+        )
+import types as _types
+import sys
+
+# we don't export these for import *, but we do want them accessible
+# as numerictypes.bool, etc.
+if sys.version_info[0] >= 3:
+    from builtins import bool, int, float, complex, object, str
+    unicode = str
+else:
+    from __builtin__ import bool, int, float, complex, object, unicode, str
+
+
+# String-handling utilities to avoid locale-dependence.
+
+# "import string" is costly to import!
+# Construct the translation tables directly
+#   "A" = chr(65), "a" = chr(97)
+_all_chars = [chr(_m) for _m in range(256)]
+_ascii_upper = _all_chars[65:65+26]
+_ascii_lower = _all_chars[97:97+26]
+LOWER_TABLE="".join(_all_chars[:65] + _ascii_lower + _all_chars[65+26:])
+UPPER_TABLE="".join(_all_chars[:97] + _ascii_upper + _all_chars[97+26:])
+
+#import string
+# assert (string.maketrans(string.ascii_uppercase, string.ascii_lowercase) == \
+#          LOWER_TABLE)
+# assert (string.maketrnas(string_ascii_lowercase, string.ascii_uppercase) == \
+#          UPPER_TABLE)
+#LOWER_TABLE = string.maketrans(string.ascii_uppercase, string.ascii_lowercase)
+#UPPER_TABLE = string.maketrans(string.ascii_lowercase, string.ascii_uppercase)
+
+def english_lower(s):
+    """ Apply English case rules to convert ASCII strings to all lower case.
+
+    This is an internal utility function to replace calls to str.lower() such
+    that we can avoid changing behavior with changing locales. In particular,
+    Turkish has distinct dotted and dotless variants of the Latin letter "I" in
+    both lowercase and uppercase. Thus, "I".lower() != "i" in a "tr" locale.
+
+    Parameters
+    ----------
+    s : str
+
+    Returns
+    -------
+    lowered : str
+
+    Examples
+    --------
+    >>> from numpy.core.numerictypes import english_lower
+    >>> english_lower('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_')
+    'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz0123456789_'
+    >>> english_lower('')
+    ''
+    """
+    lowered = s.translate(LOWER_TABLE)
+    return lowered
+
+def english_upper(s):
+    """ Apply English case rules to convert ASCII strings to all upper case.
+
+    This is an internal utility function to replace calls to str.upper() such
+    that we can avoid changing behavior with changing locales. In particular,
+    Turkish has distinct dotted and dotless variants of the Latin letter "I" in
+    both lowercase and uppercase. Thus, "i".upper() != "I" in a "tr" locale.
+
+    Parameters
+    ----------
+    s : str
+
+    Returns
+    -------
+    uppered : str
+
+    Examples
+    --------
+    >>> from numpy.core.numerictypes import english_upper
+    >>> english_upper('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_')
+    'ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
+    >>> english_upper('')
+    ''
+    """
+    uppered = s.translate(UPPER_TABLE)
+    return uppered
+
+def english_capitalize(s):
+    """ Apply English case rules to convert the first character of an ASCII
+    string to upper case.
+
+    This is an internal utility function to replace calls to str.capitalize()
+    such that we can avoid changing behavior with changing locales.
+
+    Parameters
+    ----------
+    s : str
+
+    Returns
+    -------
+    capitalized : str
+
+    Examples
+    --------
+    >>> from numpy.core.numerictypes import english_capitalize
+    >>> english_capitalize('int8')
+    'Int8'
+    >>> english_capitalize('Int8')
+    'Int8'
+    >>> english_capitalize('')
+    ''
+    """
+    if s:
+        return english_upper(s[0]) + s[1:]
+    else:
+        return s
+
+
+sctypeDict = {}      # Contains all leaf-node scalar types with aliases
+sctypeNA = {}        # Contails all leaf-node types -> numarray type equivalences
+allTypes = {}      # Collect the types we will add to the module here
+
+def _evalname(name):
+    k = 0
+    for ch in name:
+        if ch in '0123456789':
+            break
+        k += 1
+    try:
+        bits = int(name[k:])
+    except ValueError:
+        bits = 0
+    base = name[:k]
+    return base, bits
+
+def bitname(obj):
+    """Return a bit-width name for a given type object"""
+    name = obj.__name__
+    base = ''
+    char = ''
+    try:
+        if name[-1] == '_':
+            newname = name[:-1]
+        else:
+            newname = name
+        info = typeinfo[english_upper(newname)]
+        assert(info[-1] == obj)  # sanity check
+        bits = info[2]
+
+    except KeyError:     # bit-width name
+        base, bits = _evalname(name)
+        char = base[0]
+
+    if name == 'bool_':
+        char = 'b'
+        base = 'bool'
+    elif name=='void':
+        char = 'V'
+        base = 'void'
+    elif name=='object_':
+        char = 'O'
+        base = 'object'
+        bits = 0
+    elif name=='datetime64':
+        char = 'M'
+    elif name=='timedelta64':
+        char = 'm'
+
+    if sys.version_info[0] >= 3:
+        if name=='bytes_':
+            char = 'S'
+            base = 'bytes'
+        elif name=='str_':
+            char = 'U'
+            base = 'str'
+    else:
+        if name=='string_':
+            char = 'S'
+            base = 'string'
+        elif name=='unicode_':
+            char = 'U'
+            base = 'unicode'
+
+    bytes = bits // 8
+
+    if char != '' and bytes != 0:
+        char = "%s%d" % (char, bytes)
+
+    return base, bits, char
+
+
+def _add_types():
+    for a in typeinfo.keys():
+        name = english_lower(a)
+        if isinstance(typeinfo[a], tuple):
+            typeobj = typeinfo[a][-1]
+
+            # define C-name and insert typenum and typechar references also
+            allTypes[name] = typeobj
+            sctypeDict[name] = typeobj
+            sctypeDict[typeinfo[a][0]] = typeobj
+            sctypeDict[typeinfo[a][1]] = typeobj
+
+        else:  # generic class
+            allTypes[name] = typeinfo[a]
+_add_types()
+
+def _add_aliases():
+    for a in typeinfo.keys():
+        name = english_lower(a)
+        if not isinstance(typeinfo[a], tuple):
+            continue
+        typeobj = typeinfo[a][-1]
+        # insert bit-width version for this class (if relevant)
+        base, bit, char = bitname(typeobj)
+        if base[-3:] == 'int' or char[0] in 'ui': continue
+        if base != '':
+            myname = "%s%d" % (base, bit)
+            if (name != 'longdouble' and name != 'clongdouble') or \
+                   myname not in allTypes.keys():
+                allTypes[myname] = typeobj
+                sctypeDict[myname] = typeobj
+                if base == 'complex':
+                    na_name = '%s%d' % (english_capitalize(base), bit//2)
+                elif base == 'bool':
+                    na_name = english_capitalize(base)
+                    sctypeDict[na_name] = typeobj
+                else:
+                    na_name = "%s%d" % (english_capitalize(base), bit)
+                    sctypeDict[na_name] = typeobj
+                sctypeNA[na_name] = typeobj
+                sctypeDict[na_name] = typeobj
+                sctypeNA[typeobj] = na_name
+                sctypeNA[typeinfo[a][0]] = na_name
+        if char != '':
+            sctypeDict[char] = typeobj
+            sctypeNA[char] = na_name
+_add_aliases()
+
+# Integers handled so that
+# The int32, int64 types should agree exactly with
+#  PyArray_INT32, PyArray_INT64 in C
+# We need to enforce the same checking as is done
+#  in arrayobject.h where the order of getting a
+#  bit-width match is:
+#       long, longlong, int, short, char
+#   for int8, int16, int32, int64, int128
+
+def _add_integer_aliases():
+    _ctypes = ['LONG', 'LONGLONG', 'INT', 'SHORT', 'BYTE']
+    for ctype in _ctypes:
+        val = typeinfo[ctype]
+        bits = val[2]
+        charname = 'i%d' % (bits//8,)
+        ucharname = 'u%d' % (bits//8,)
+        intname = 'int%d' % bits
+        UIntname = 'UInt%d' % bits
+        Intname = 'Int%d' % bits
+        uval = typeinfo['U'+ctype]
+        typeobj = val[-1]
+        utypeobj = uval[-1]
+        if intname not in allTypes.keys():
+            uintname = 'uint%d' % bits
+            allTypes[intname] = typeobj
+            allTypes[uintname] = utypeobj
+            sctypeDict[intname] = typeobj
+            sctypeDict[uintname] = utypeobj
+            sctypeDict[Intname] = typeobj
+            sctypeDict[UIntname] = utypeobj
+            sctypeDict[charname] = typeobj
+            sctypeDict[ucharname] = utypeobj
+            sctypeNA[Intname] = typeobj
+            sctypeNA[UIntname] = utypeobj
+            sctypeNA[charname] = typeobj
+            sctypeNA[ucharname] = utypeobj
+        sctypeNA[typeobj] = Intname
+        sctypeNA[utypeobj] = UIntname
+        sctypeNA[val[0]] = Intname
+        sctypeNA[uval[0]] = UIntname
+_add_integer_aliases()
+
+# We use these later
+void = allTypes['void']
+generic = allTypes['generic']
+
+#
+# Rework the Python names (so that float and complex and int are consistent
+#                            with Python usage)
+#
+def _set_up_aliases():
+    type_pairs = [('complex_', 'cdouble'),
+                  ('int0', 'intp'),
+                  ('uint0', 'uintp'),
+                  ('single', 'float'),
+                  ('csingle', 'cfloat'),
+                  ('singlecomplex', 'cfloat'),
+                  ('float_', 'double'),
+                  ('intc', 'int'),
+                  ('uintc', 'uint'),
+                  ('int_', 'long'),
+                  ('uint', 'ulong'),
+                  ('cfloat', 'cdouble'),
+                  ('bool_', 'bool'),
+                  ('unicode_', 'unicode'),
+                  ]
+    if sys.version_info[0] >= 3:
+        type_pairs.extend([('bytes_', 'string'),
+                           ('str_', 'unicode'),
+                           ('string_', 'string')])
+    else:
+        type_pairs.extend([('str_', 'string'),
+                           ('string_', 'string'),
+                           ('bytes_', 'string')])
+    for alias, t in type_pairs:
+        allTypes[alias] = allTypes[t]
+        sctypeDict[alias] = sctypeDict[t]
+    # Remove aliases overriding python types and modules
+    to_remove = ['ulong', 'object', 'unicode', 'int', 'long', 'float',
+                 'complex', 'bool', 'string', 'datetime', 'timedelta']
+    if sys.version_info[0] >= 3:
+        # Py3K
+        to_remove.append('bytes')
+        to_remove.append('str')
+        to_remove.remove('unicode')
+        to_remove.remove('long')
+    for t in to_remove:
+        try:
+            del allTypes[t]
+            del sctypeDict[t]
+        except KeyError:
+            pass
+_set_up_aliases()
+
+# Now, construct dictionary to lookup character codes from types
+_sctype2char_dict = {}
+def _construct_char_code_lookup():
+    for name in typeinfo.keys():
+        tup = typeinfo[name]
+        if isinstance(tup, tuple):
+            if tup[0] not in ['p', 'P']:
+                _sctype2char_dict[tup[-1]] = tup[0]
+_construct_char_code_lookup()
+
+
+sctypes = {'int': [],
+           'uint':[],
+           'float':[],
+           'complex':[],
+           'others':[bool, object, str, unicode, void]}
+
+def _add_array_type(typename, bits):
+    try:
+        t = allTypes['%s%d' % (typename, bits)]
+    except KeyError:
+        pass
+    else:
+        sctypes[typename].append(t)
+
+def _set_array_types():
+    ibytes = [1, 2, 4, 8, 16, 32, 64]
+    fbytes = [2, 4, 8, 10, 12, 16, 32, 64]
+    for bytes in ibytes:
+        bits = 8*bytes
+        _add_array_type('int', bits)
+        _add_array_type('uint', bits)
+    for bytes in fbytes:
+        bits = 8*bytes
+        _add_array_type('float', bits)
+        _add_array_type('complex', 2*bits)
+    _gi = dtype('p')
+    if _gi.type not in sctypes['int']:
+        indx = 0
+        sz = _gi.itemsize
+        _lst = sctypes['int']
+        while (indx < len(_lst) and sz >= _lst[indx](0).itemsize):
+            indx += 1
+        sctypes['int'].insert(indx, _gi.type)
+        sctypes['uint'].insert(indx, dtype('P').type)
+_set_array_types()
+
+
+genericTypeRank = ['bool', 'int8', 'uint8', 'int16', 'uint16',
+                   'int32', 'uint32', 'int64', 'uint64', 'int128',
+                   'uint128', 'float16',
+                   'float32', 'float64', 'float80', 'float96', 'float128',
+                   'float256',
+                   'complex32', 'complex64', 'complex128', 'complex160',
+                   'complex192', 'complex256', 'complex512', 'object']
+
+def maximum_sctype(t):
+    """
+    Return the scalar type of highest precision of the same kind as the input.
+
+    Parameters
+    ----------
+    t : dtype or dtype specifier
+        The input data type. This can be a `dtype` object or an object that
+        is convertible to a `dtype`.
+
+    Returns
+    -------
+    out : dtype
+        The highest precision data type of the same kind (`dtype.kind`) as `t`.
+
+    See Also
+    --------
+    obj2sctype, mintypecode, sctype2char
+    dtype
+
+    Examples
+    --------
+    >>> np.maximum_sctype(np.int)
+    <type 'numpy.int64'>
+    >>> np.maximum_sctype(np.uint8)
+    <type 'numpy.uint64'>
+    >>> np.maximum_sctype(np.complex)
+    <type 'numpy.complex192'>
+
+    >>> np.maximum_sctype(str)
+    <type 'numpy.string_'>
+
+    >>> np.maximum_sctype('i2')
+    <type 'numpy.int64'>
+    >>> np.maximum_sctype('f4')
+    <type 'numpy.float96'>
+
+    """
+    g = obj2sctype(t)
+    if g is None:
+        return t
+    t = g
+    name = t.__name__
+    base, bits = _evalname(name)
+    if bits == 0:
+        return t
+    else:
+        return sctypes[base][-1]
+
+try:
+    buffer_type = _types.BufferType
+except AttributeError:
+    # Py3K
+    buffer_type = memoryview
+
+_python_types = {int: 'int_',
+                 float: 'float_',
+                 complex: 'complex_',
+                 bool: 'bool_',
+                 bytes: 'bytes_',
+                 unicode: 'unicode_',
+                 buffer_type: 'void',
+                }
+
+if sys.version_info[0] >= 3:
+    def _python_type(t):
+        """returns the type corresponding to a certain Python type"""
+        if not isinstance(t, type):
+            t = type(t)
+        return allTypes[_python_types.get(t, 'object_')]
+else:
+    def _python_type(t):
+        """returns the type corresponding to a certain Python type"""
+        if not isinstance(t, _types.TypeType):
+            t = type(t)
+        return allTypes[_python_types.get(t, 'object_')]
+
+def issctype(rep):
+    """
+    Determines whether the given object represents a scalar data-type.
+
+    Parameters
+    ----------
+    rep : any
+        If `rep` is an instance of a scalar dtype, True is returned. If not,
+        False is returned.
+
+    Returns
+    -------
+    out : bool
+        Boolean result of check whether `rep` is a scalar dtype.
+
+    See Also
+    --------
+    issubsctype, issubdtype, obj2sctype, sctype2char
+
+    Examples
+    --------
+    >>> np.issctype(np.int32)
+    True
+    >>> np.issctype(list)
+    False
+    >>> np.issctype(1.1)
+    False
+
+    Strings are also a scalar type:
+
+    >>> np.issctype(np.dtype('str'))
+    True
+
+    """
+    if not isinstance(rep, (type, dtype)):
+        return False
+    try:
+        res = obj2sctype(rep)
+        if res and res != object_:
+            return True
+        return False
+    except:
+        return False
+
+def obj2sctype(rep, default=None):
+    """
+    Return the scalar dtype or NumPy equivalent of Python type of an object.
+
+    Parameters
+    ----------
+    rep : any
+        The object of which the type is returned.
+    default : any, optional
+        If given, this is returned for objects whose types can not be
+        determined. If not given, None is returned for those objects.
+
+    Returns
+    -------
+    dtype : dtype or Python type
+        The data type of `rep`.
+
+    See Also
+    --------
+    sctype2char, issctype, issubsctype, issubdtype, maximum_sctype
+
+    Examples
+    --------
+    >>> np.obj2sctype(np.int32)
+    <type 'numpy.int32'>
+    >>> np.obj2sctype(np.array([1., 2.]))
+    <type 'numpy.float64'>
+    >>> np.obj2sctype(np.array([1.j]))
+    <type 'numpy.complex128'>
+
+    >>> np.obj2sctype(dict)
+    <type 'numpy.object_'>
+    >>> np.obj2sctype('string')
+    <type 'numpy.string_'>
+
+    >>> np.obj2sctype(1, default=list)
+    <type 'list'>
+
+    """
+    try:
+        if issubclass(rep, generic):
+            return rep
+    except TypeError:
+        pass
+    if isinstance(rep, dtype):
+        return rep.type
+    if isinstance(rep, type):
+        return _python_type(rep)
+    if isinstance(rep, ndarray):
+        return rep.dtype.type
+    try:
+        res = dtype(rep)
+    except:
+        return default
+    return res.type
+
 
 def issubclass_(arg1, arg2):
     """
@@ -40,6 +698,36 @@
     except TypeError:
         return False
 
+def issubsctype(arg1, arg2):
+    """
+    Determine if the first argument is a subclass of the second argument.
+
+    Parameters
+    ----------
+    arg1, arg2 : dtype or dtype specifier
+        Data-types.
+
+    Returns
+    -------
+    out : bool
+        The result.
+
+    See Also
+    --------
+    issctype, issubdtype,obj2sctype
+
+    Examples
+    --------
+    >>> np.issubsctype('S8', str)
+    True
+    >>> np.issubsctype(np.array([1]), np.int)
+    True
+    >>> np.issubsctype(np.array([1]), np.float)
+    False
+
+    """
+    return issubclass(obj2sctype(arg1), obj2sctype(arg2))
+
 def issubdtype(arg1, arg2):
     """
     Returns True if first argument is a typecode lower/equal in type hierarchy.
@@ -74,3 +762,270 @@
     else:
         val = mro[0]
     return issubclass(dtype(arg1).type, val)
+
+
+# This dictionary allows look up based on any alias for an array data-type
+class _typedict(dict):
+    """
+    Base object for a dictionary for look-up with any alias for an array dtype.
+
+    Instances of `_typedict` can not be used as dictionaries directly,
+    first they have to be populated.
+
+    """
+    def __getitem__(self, obj):
+        return dict.__getitem__(self, obj2sctype(obj))
+
+nbytes = _typedict()
+_alignment = _typedict()
+_maxvals = _typedict()
+_minvals = _typedict()
+def _construct_lookups():
+    for name, val in typeinfo.items():
+        if not isinstance(val, tuple):
+            continue
+        obj = val[-1]
+        nbytes[obj] = val[2] // 8
+        _alignment[obj] = val[3]
+        if (len(val) > 5):
+            _maxvals[obj] = val[4]
+            _minvals[obj] = val[5]
+        else:
+            _maxvals[obj] = None
+            _minvals[obj] = None
+
+_construct_lookups()
+
+def sctype2char(sctype):
+    """
+    Return the string representation of a scalar dtype.
+
+    Parameters
+    ----------
+    sctype : scalar dtype or object
+        If a scalar dtype, the corresponding string character is
+        returned. If an object, `sctype2char` tries to infer its scalar type
+        and then return the corresponding string character.
+
+    Returns
+    -------
+    typechar : str
+        The string character corresponding to the scalar type.
+
+    Raises
+    ------
+    ValueError
+        If `sctype` is an object for which the type can not be inferred.
+
+    See Also
+    --------
+    obj2sctype, issctype, issubsctype, mintypecode
+
+    Examples
+    --------
+    >>> for sctype in [np.int32, np.float, np.complex, np.string_, np.ndarray]:
+    ...     print np.sctype2char(sctype)
+    l
+    d
+    D
+    S
+    O
+
+    >>> x = np.array([1., 2-1.j])
+    >>> np.sctype2char(x)
+    'D'
+    >>> np.sctype2char(list)
+    'O'
+
+    """
+    sctype = obj2sctype(sctype)
+    if sctype is None:
+        raise ValueError("unrecognized type")
+    return _sctype2char_dict[sctype]
+
+# Create dictionary of casting functions that wrap sequences
+# indexed by type or type character
+
+
+cast = _typedict()
+try:
+    ScalarType = [_types.IntType, _types.FloatType, _types.ComplexType,
+                  _types.LongType, _types.BooleanType,
+                   _types.StringType, _types.UnicodeType, _types.BufferType]
+except AttributeError:
+    # Py3K
+    ScalarType = [int, float, complex, int, bool, bytes, str, memoryview]
+
+ScalarType.extend(_sctype2char_dict.keys())
+ScalarType = tuple(ScalarType)
+for key in _sctype2char_dict.keys():
+    cast[key] = lambda x, k=key : array(x, copy=False).astype(k)
+
+# Create the typestring lookup dictionary
+_typestr = _typedict()
+for key in _sctype2char_dict.keys():
+    if issubclass(key, allTypes['flexible']):
+        _typestr[key] = _sctype2char_dict[key]
+    else:
+        _typestr[key] = empty((1,), key).dtype.str[1:]
+
+# Make sure all typestrings are in sctypeDict
+for key, val in _typestr.items():
+    if val not in sctypeDict:
+        sctypeDict[val] = key
+
+# Add additional strings to the sctypeDict
+
+if sys.version_info[0] >= 3:
+    _toadd = ['int', 'float', 'complex', 'bool',
+              'str', 'bytes', ('a', allTypes['bytes_'])]
+else:
+    _toadd = ['int', 'float', 'complex', 'bool', 'string',
+              ('str', allTypes['string_']),
+              'unicode', ('a', allTypes['string_'])]
+
+for name in _toadd:
+    if isinstance(name, tuple):
+        sctypeDict[name[0]] = name[1]
+    else:
+        sctypeDict[name] = allTypes['%s_' % name]
+
+del _toadd, name
+
+# Now add the types we've determined to this module
+for key in allTypes:
+    globals()[key] = allTypes[key]
+    __all__.append(key)
+
+del key
+
+typecodes = {'Character':'c',
+             'Integer':'bhilqp',
+             'UnsignedInteger':'BHILQP',
+             'Float':'efdg',
+             'Complex':'FDG',
+             'AllInteger':'bBhHiIlLqQpP',
+             'AllFloat':'efdgFDG',
+             'Datetime': 'Mm',
+             'All':'?bhilqpBHILQPefdgFDGSUVOMm'}
+
+# backwards compatibility --- deprecated name
+typeDict = sctypeDict
+typeNA = sctypeNA
+
+# b -> boolean
+# u -> unsigned integer
+# i -> signed integer
+# f -> floating point
+# c -> complex
+# M -> datetime
+# m -> timedelta
+# S -> string
+# U -> Unicode string
+# V -> record
+# O -> Python object
+_kind_list = ['b', 'u', 'i', 'f', 'c', 'S', 'U', 'V', 'O', 'M', 'm']
+
+__test_types = '?'+typecodes['AllInteger'][:-2]+typecodes['AllFloat']+'O'
+__len_test_types = len(__test_types)
+
+# Keep incrementing until a common type both can be coerced to
+#  is found.  Otherwise, return None
+def _find_common_coerce(a, b):
+    if a > b:
+        return a
+    try:
+        thisind = __test_types.index(a.char)
+    except ValueError:
+        return None
+    return _can_coerce_all([a, b], start=thisind)
+
+# Find a data-type that all data-types in a list can be coerced to
+def _can_coerce_all(dtypelist, start=0):
+    N = len(dtypelist)
+    if N == 0:
+        return None
+    if N == 1:
+        return dtypelist[0]
+    thisind = start
+    while thisind < __len_test_types:
+        newdtype = dtype(__test_types[thisind])
+        numcoerce = len([x for x in dtypelist if newdtype >= x])
+        if numcoerce == N:
+            return newdtype
+        thisind += 1
+    return None
+
+def find_common_type(array_types, scalar_types):
+    """
+    Determine common type following standard coercion rules.
+
+    Parameters
+    ----------
+    array_types : sequence
+        A list of dtypes or dtype convertible objects representing arrays.
+    scalar_types : sequence
+        A list of dtypes or dtype convertible objects representing scalars.
+
+    Returns
+    -------
+    datatype : dtype
+        The common data type, which is the maximum of `array_types` ignoring
+        `scalar_types`, unless the maximum of `scalar_types` is of a
+        different kind (`dtype.kind`). If the kind is not understood, then
+        None is returned.
+
+    See Also
+    --------
+    dtype, common_type, can_cast, mintypecode
+
+    Examples
+    --------
+    >>> np.find_common_type([], [np.int64, np.float32, np.complex])
+    dtype('complex128')
+    >>> np.find_common_type([np.int64, np.float32], [])
+    dtype('float64')
+
+    The standard casting rules ensure that a scalar cannot up-cast an
+    array unless the scalar is of a fundamentally different kind of data
+    (i.e. under a different hierarchy in the data type hierarchy) then
+    the array:
+
+    >>> np.find_common_type([np.float32], [np.int64, np.float64])
+    dtype('float32')
+
+    Complex is of a different type, so it up-casts the float in the
+    `array_types` argument:
+
+    >>> np.find_common_type([np.float32], [np.complex])
+    dtype('complex128')
+
+    Type specifier strings are convertible to dtypes and can therefore
+    be used instead of dtypes:
+
+    >>> np.find_common_type(['f4', 'f4', 'i4'], ['c8'])
+    dtype('complex128')
+
+    """
+    array_types = [dtype(x) for x in array_types]
+    scalar_types = [dtype(x) for x in scalar_types]
+
+    maxa = _can_coerce_all(array_types)
+    maxsc = _can_coerce_all(scalar_types)
+
+    if maxa is None:
+        return maxsc
+
+    if maxsc is None:
+        return maxa
+
+    try:
+        index_a = _kind_list.index(maxa.kind)
+        index_sc = _kind_list.index(maxsc.kind)
+    except ValueError:
+        return None
+
+    if index_sc > index_a:
+        return _find_common_coerce(maxsc, maxa)
+    else:
+        return maxa
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -26,74 +26,6 @@
     }
 
 
-class NumericTypesModule(MixedModule):
-    appleveldefs = {}
-    interpleveldefs = {
-        'generic': 'interp_boxes.W_GenericBox',
-        'number': 'interp_boxes.W_NumberBox',
-        'integer': 'interp_boxes.W_IntegerBox',
-        'signedinteger': 'interp_boxes.W_SignedIntegerBox',
-        'unsignedinteger': 'interp_boxes.W_UnsignedIntegerBox',
-        'bool_': 'interp_boxes.W_BoolBox',
-        'bool8': 'interp_boxes.W_BoolBox',
-        'int8': 'interp_boxes.W_Int8Box',
-        'byte': 'interp_boxes.W_Int8Box',
-        'uint8': 'interp_boxes.W_UInt8Box',
-        'ubyte': 'interp_boxes.W_UInt8Box',
-        'int16': 'interp_boxes.W_Int16Box',
-        'short': 'interp_boxes.W_Int16Box',
-        'uint16': 'interp_boxes.W_UInt16Box',
-        'ushort': 'interp_boxes.W_UInt16Box',
-        'int32': 'interp_boxes.W_Int32Box',
-        'intc': 'interp_boxes.W_Int32Box',
-        'uint32': 'interp_boxes.W_UInt32Box',
-        'uintc': 'interp_boxes.W_UInt32Box',
-        'int64': 'interp_boxes.W_Int64Box',
-        'uint64': 'interp_boxes.W_UInt64Box',
-        'longlong': 'interp_boxes.W_LongLongBox',
-        'ulonglong': 'interp_boxes.W_ULongLongBox',
-        'int_': 'interp_boxes.W_LongBox',
-        'inexact': 'interp_boxes.W_InexactBox',
-        'floating': 'interp_boxes.W_FloatingBox',
-        'float_': 'interp_boxes.W_Float64Box',
-        'float16': 'interp_boxes.W_Float16Box',
-        'float32': 'interp_boxes.W_Float32Box',
-        'float64': 'interp_boxes.W_Float64Box',
-        'intp': 'types.IntP.BoxType',
-        'uintp': 'types.UIntP.BoxType',
-        'flexible': 'interp_boxes.W_FlexibleBox',
-        'character': 'interp_boxes.W_CharacterBox',
-        'str_': 'interp_boxes.W_StringBox',
-        'string_': 'interp_boxes.W_StringBox',
-        'unicode_': 'interp_boxes.W_UnicodeBox',
-        'void': 'interp_boxes.W_VoidBox',
-        'complexfloating': 'interp_boxes.W_ComplexFloatingBox',
-        'complex_': 'interp_boxes.W_Complex128Box',
-        'complex128': 'interp_boxes.W_Complex128Box',
-        'complex64': 'interp_boxes.W_Complex64Box',
-        'cfloat': 'interp_boxes.W_Complex64Box',
-    }
-    if ENABLED_LONG_DOUBLE:
-        long_double_dtypes = [
-            ('longdouble', 'interp_boxes.W_LongDoubleBox'),
-            ('longfloat', 'interp_boxes.W_LongDoubleBox'),
-            ('clongdouble', 'interp_boxes.W_CLongDoubleBox'),
-            ('clongfloat', 'interp_boxes.W_CLongDoubleBox'),
-        ]
-        if long_double_size == 16:
-            long_double_dtypes += [
-                ('float128', 'interp_boxes.W_Float128Box'),
-                ('complex256', 'interp_boxes.W_Complex256Box'),
-            ]
-        elif long_double_size == 12:
-            long_double_dtypes += [
-                ('float96', 'interp_boxes.W_Float96Box'),
-                ('complex192', 'interp_boxes.W_Complex192Box'),
-            ]
-        for dt, box in long_double_dtypes:
-            interpleveldefs[dt] = box
-
-
 class UMathModule(MixedModule):
     appleveldefs = {}
     interpleveldefs = {}
@@ -192,6 +124,5 @@
     }
     submodules = {
         'multiarray': MultiArrayModule,
-        'numerictypes': NumericTypesModule,
         'umath': UMathModule,
     }

From noreply at buildbot.pypy.org  Tue Oct 15 11:04:10 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 11:04:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test for complex aliases
Message-ID: <20131015090410.48C9B1D22CC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67388:c7965a33a50a
Date: 2013-10-15 05:00 -0400
http://bitbucket.org/pypy/pypy/changeset/c7965a33a50a/

Log:	fix test for complex aliases

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -585,7 +585,8 @@
         import numpypy as numpy
 
         assert numpy.complex_ is numpy.complex128
-        assert numpy.cfloat is numpy.complex64
+        assert numpy.csingle is numpy.complex64
+        assert numpy.cfloat is numpy.complex128
         assert numpy.complex64.__mro__ == (numpy.complex64,
             numpy.complexfloating, numpy.inexact, numpy.number, numpy.generic,
             object)

From noreply at buildbot.pypy.org  Tue Oct 15 11:04:11 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 11:04:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: unused exports
Message-ID: <20131015090411.6E7D91C135D@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67389:bc2983f3a787
Date: 2013-10-15 05:02 -0400
http://bitbucket.org/pypy/pypy/changeset/bc2983f3a787/

Log:	unused exports

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -117,11 +117,7 @@
 class Module(MixedModule):
     applevel_name = '_numpypy'
     appleveldefs = {}
-    interpleveldefs = {
-        'choose': 'interp_arrayops.choose',
-        'put': 'interp_arrayops.put',
-        'repeat': 'interp_arrayops.repeat',
-    }
+    interpleveldefs = {}
     submodules = {
         'multiarray': MultiArrayModule,
         'umath': UMathModule,

From noreply at buildbot.pypy.org  Tue Oct 15 11:22:12 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 15 Oct 2013 11:22:12 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: try to explain that PyPy's
 approach to GC is better than reference counting. A lot of people are
 surprised to read that generational GC is faster than refcount
Message-ID: <20131015092212.CBD871C0203@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: extradoc
Changeset: r5081:6b5bf67ab352
Date: 2013-10-15 11:22 +0200
http://bitbucket.org/pypy/extradoc/changeset/6b5bf67ab352/

Log:	try to explain that PyPy's approach to GC is better than reference
	counting. A lot of people are surprised to read that generational GC
	is faster than refcount

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -34,8 +34,13 @@
 
 PyPy essentially has only the cycle finder - it does not bother with reference
 counting, instead it walks alive objects every now and then (this is a big
-simplification, PyPy's GC is much more complex than this). As a result it also
-has the problem of GC pauses. To alleviate this problem, which is essential for
+simplification, PyPy's GC is much more complex than this). Although this might
+sound like a missing feature, it is really one of the reasons why PyPy is so
+fast, because at the end of the day the total time spent in managing the
+memory is lower in PyPy than CPython. However, as a result, PyPy also has the
+problem of GC pauses.
+
+To alleviate this problem, which is essential for
 applications like games, we started to work on incremental GC, which spreads
 the walking of objects and cleaning them across the execution time in smaller
 intervals. The work was sponsored by the Raspberry Pi foundation, started

From noreply at buildbot.pypy.org  Tue Oct 15 11:24:56 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 15 Oct 2013 11:24:56 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: memusage.png
Message-ID: <20131015092456.DD3531C0203@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r5082:d838dd566786
Date: 2013-10-15 11:24 +0200
http://bitbucket.org/pypy/extradoc/changeset/d838dd566786/

Log:	memusage.png

diff --git a/blog/draft/memusage.png b/blog/draft/memusage.png
new file mode 100644
index 0000000000000000000000000000000000000000..146c216f24d663194591a701380cff0d5c718461
GIT binary patch

[cut]


From noreply at buildbot.pypy.org  Tue Oct 15 11:28:41 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 15 Oct 2013 11:28:41 +0200 (CEST)
Subject: [pypy-commit] stmgc default: remove copy-paste left-over
Message-ID: <20131015092841.8D93C1C0203@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r536:f53edba476a6
Date: 2013-10-15 11:28 +0200
http://bitbucket.org/pypy/stmgc/changeset/f53edba476a6/

Log:	remove copy-paste left-over

diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -23,7 +23,7 @@
         i++;
     }
     cur += sprintf(cur, "tid=%ld", stm_get_tid(obj));
-    cur += sprintf(cur, " : rev=%lx : orig=%lx", 
+    cur += sprintf(cur, " : rev=0x%lx : orig=0x%lx", 
                    (long)obj->h_revision, (long)obj->h_original);
     return tmp_buf;
 }
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -222,14 +222,14 @@
     }
     else {
         CommitTransaction();
-        if (d->active != 2) {
-            unsigned long limit = d->reads_size_limit_nonatomic;
-            if (limit != 0 && limit < (stm_regular_length_limit >> 1))
-                limit = (limit << 1) | 1;
-            else
-                limit = stm_regular_length_limit;
-            d->reads_size_limit_nonatomic = limit;
-        }
+
+        unsigned long limit = d->reads_size_limit_nonatomic;
+        if (limit != 0 && limit < (stm_regular_length_limit >> 1))
+            limit = (limit << 1) | 1;
+        else
+            limit = stm_regular_length_limit;
+        d->reads_size_limit_nonatomic = limit;
+
         stm_begin_transaction(buf, longjmp_callback);
     }
 }

From noreply at buildbot.pypy.org  Tue Oct 15 11:32:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 15 Oct 2013 11:32:00 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Complete the last section
Message-ID: <20131015093200.E26D71C0203@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5083:51a7f13ee794
Date: 2013-10-15 11:31 +0200
http://bitbucket.org/pypy/extradoc/changeset/51a7f13ee794/

Log:	Complete the last section

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -124,72 +124,106 @@
 Nitty gritty details
 ====================
 
-XXX insert some links where you can read about terms used
+Here are some nitty gritty details for people really interested in
+Garbage Collection.  This was done as a patch to "minimark", our current
+GC, and called "incminimark" for now.  The former is a generational
+stop-the-world GC.  New objects are allocated "young", which means that
+they initially live in the "nursery", a special zone of a few MB of
+memory.  When the nursery is full, a "minor collection" step moves the
+surviving objects out of the nursery.  This can be done quickly (a few
+millisecond) because we only need to walk through the young objects that
+survive --- usually a small fraction of all young objects; and also by
+far not *all* objects that are alive at this point, but only the young
+ones.  However, from time to time this minor collection is followed by a
+"major collection": in that step, we really need to walk all objects to
+classify which ones are still alive and which ones are now dead
+("marking") and free the memory occupied by the dead ones ("sweeping").
+You can read more details here__.
 
-This was done as a patch to "minimark", our current GC, and called
-"incminimark" for now.  The former is a generational stop-the-world GC.
-New objects are allocated "young", i.e. in the nursery, a special zone
-of a few MB of memory.  When it is full, a "minor collection" step moves
-the surviving objects out of the nursery.  This can be done quickly (a
-few millisecond at most) because we only need to walk through the young
-objects that survive --- usually a small fraction of all young objects.
-From time to time, this minor collection is followed by a "major
-collection": in that step, we walk *all* objects to classify which ones
-are still alive and which ones are now dead (*marking*) and free the
-memory occupied by the dead ones (*sweeping*).
+.. __: http://doc.pypy.org/en/latest/garbage_collection.html#minimark-gc
 
 This "major collection" is what gives the long GC pauses.  To fix this
 problem we made the GC incremental: instead of running one complete
-major collection, we split its work into a variable number of pieces
-and run each piece after every minor collection for a while, until there
-are no more pieces.  The pieces are each doing a fraction of marking, or
-a fraction of sweeping.
+major collection, we split its work into a variable number of pieces and
+run each piece after every minor collection for a while, until there are
+no more pieces.  The pieces are each doing a fraction of marking, or a
+fraction of sweeping.  It adds some few milliseconds after each of these
+minor collections, rather than requiring hundreds of milliseconds in one
+go.
 
 The main issue is that splitting the major collections means that the
-main program is actually running between the pieces, and so can change
-the pointers in the objects to point to other objects.  This is not
-a problem for sweeping: dead objects will remain dead whatever the main
-program does.  However, it is a problem for marking.  Let us see why.
+main program is actually running between the pieces, and so it can
+change the pointers in the objects to point to other objects.  This is
+not a problem for sweeping: dead objects will remain dead whatever the
+main program does.  However, it is a problem for marking.  Let us see
+why.
+
+.. __: http://rubini.us/2013/06/22/concurrent-garbage-collection/
+.. __: http://wiki.luajit.org/New-Garbage-Collector/01fd5e5ca4f95d45e0c4b8a98b49f2b656cc23dd
 
 In terms of the incremental GC literature, objects are either "white",
-"gray" or "black".  They start as "white", become "gray" when they are
-found to be alive, and become "black" when they have been fully
-traversed --- at which point the objects that it points to have
-themselves been marked gray, or maybe are already black.  The gray
-objects are the "frontier" between the black objects that we have found
-to be reachable, and the white objects that represent the unknown part
-of the world.  When there are no more gray objects, the process is
-finished: all remaining white objects are unreachable and can be freed
-(by the following sweeping phase).
+"gray" or "black".  This is called *tri-color marking.*  See for example
+this `blog post about Rubinius`__, or this `page about LuaJIT`__.  The
+objects start as "white" at the beginning of marking; become "gray" when
+they are found to be alive; and become "black" when they have been fully
+traversed.  Marking proceeds by scanning grey objects for pointers to
+white objects.  The white objects found are turned grey, and the grey
+objects scanned are turned black.  When there are no more grey objects,
+the marking phase is complete: all remaining white objects are truly
+unreachable and can be freed (by the following sweeping phase).
 
 In this model, the important part is that a black object can never point
 to a white object: if the latter remains white until the end, it will be
 freed, which is incorrect because the black object itself can still be
-reached.
+reached.  How do we ensure that the main program, running in the middle
+of marking, will not try to write a pointer to white object into a black
+object?  This requires a "write barrier", i.e. a piece of code that runs
+every time we set a pointer into an object or array.  This piece of code
+checks if some (hopefully rare) condition is met, and calls a function
+if that is the case.
 
 The trick we used in PyPy is to consider minor collections as part of
 the whole, rather than focus only on major collections.  The existing
-minimark GC had always used a "write barrier" (a piece of code run every time
-you set or get a pointer from an object or array) to do its job, like any
-generational GC.  This write barrier is used to detect when an old
-object (outside the nursery) is modified to point to a young object
-(inside the nursery), which is essential information for minor
+minimark GC had always used a write barrier of its own to do its job,
+like any generational GC.  This existing write barrier is used to detect
+when an old object (outside the nursery) is modified to point to a young
+object (inside the nursery), which is essential information for minor
 collections.  Actually, although this was the goal, the actual write
-barrier code was simpler: it just recorded all old objects into which we
-wrote *any* pointer --- to a young or old object.  It is actually a
-performance improvement, because we don't need to check over and over
-again if the written pointer points to a young object or not.
+barrier code is simpler: it just records all old objects into which we
+write *any* pointer --- to a young or old object.  As we found out over
+time, doing so is not actually slower, and might actually be a
+performance improvement: for example, if the main program does a lot of
+writes into the same old object, we don't need to check over and over
+again if the written pointer points to a young object or not.  We just
+record the old object in some list the first time, and that's it.
 
-This *unmodified* write barrier works for incminimark too.  Imagine that
-we are in the middle of the marking phase, running the main program.
-The write barrier will record all old objects that are being modified.
-Then at the next minor collection, all surviving young objects will be
-moved out of the nursery.  At this point, as we're about to continue
-running the major collection's marking phase, we simply add to the list
-of pending gray objects all the objects that we consider --- both the
-objects listed as "old objects that are being modified", and the objects
-that we just moved out of the nursery.  A fraction of the former list
-are turned back from the black to the gray color.  This technique
+The trick is that this *unmodified* write barrier works for incminimark
+too.  Imagine that we are in the middle of the marking phase, running
+the main program.  The write barrier will record all old objects that
+are being modified.  Then at the next minor collection, all surviving
+young objects will be moved out of the nursery.  At this point, as we're
+about to continue running the major collection's marking phase, we
+simply add to the list of pending gray objects all the objects that we
+just considered --- both the objects listed as "old objects that are
+being modified", and the objects that we just moved out of the nursery.
+A fraction from the former list were black object; so this mean that
+they are turned back from the black to the gray color.  This technique
 implements nicely, if indirectly, what is called a "backward write
-barrier" in the literature: the backwardness is about the color that
-occasionally progresses backward from black to gray.
+barrier" in the literature.  The backwardness is about the color that
+needs to be changed in the opposite of the usual direction "white ->
+gray -> black", thus making more work for the GC.  (This is as opposed
+to "forward write barrier", where we would also detect "black -> white"
+writes but turn the white object gray.)
+
+In summary, I realize that this description is less about how we turned
+minimark into incminimark, and more about how we differ from the
+standard way of making a GC incremental.  What we really had to do to
+make incminimark was to write logic that says "if the major collection
+is in the middle of the marking phase, then add this object to the list
+of gray objects", and put it at a few places throughout minor
+collection.  Then we simply split a major collection into increments,
+doing marking or sweeping of some (relatively arbitrary) number of
+objects before returning.  That's why, after we found that the existing
+write barrier would do, it was not much actual work, and could be done
+without major changes.  For example, not a single line from the JIT
+needed adaptation.  All in all it was relatively painless work. ``:-)``

From noreply at buildbot.pypy.org  Tue Oct 15 11:32:46 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 15 Oct 2013 11:32:46 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: update TODO
Message-ID: <20131015093246.2A4EB1C0203@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67390:bafcb0cdff48
Date: 2013-10-15 10:35 +0200
http://bitbucket.org/pypy/pypy/changeset/bafcb0cdff48/

Log:	update TODO

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,7 +1,3 @@
-------------------------------------------------------------
-
-kill INEVITABLE in et.c, replace with "global_cur_time & 1" again
-
 ------------------------------------------------------------
 
 try to let non-atomic inevitable transactions run for longer, until
@@ -21,11 +17,7 @@
 
 ------------------------------------------------------------
 
-implement thread-locals in RPython (for the executioncontext)
-
-------------------------------------------------------------
-
-optimize the static placement of the STM_XxxBARRIERs
+optimize the static placement of the STM_XxxBARRIERs and use them in JIT
 
 ------------------------------------------------------------
 
@@ -39,8 +31,7 @@
 to the PyFrame object (make sure it's always written and don't put more
 barriers)
 
-in parallel, tweak the API of stmgc: think about adding
-stm_repeat_read_barrier, and support "tentative" write_barrier calls
+in parallel, tweak the API of stmgc: support "tentative" write_barrier calls
 that are not actually followed by a write (checked by comparing the
 object contents)
 
@@ -58,4 +49,16 @@
 fast-path, as well as splitting it based e.g. on the RPython type of
 object.  See also vtune.
 
-reimplement the fast-path of the nursery allocations in the GC
+JIT
+~~~
+
+* reimplement the fast-path of the nursery allocations in the GC
+** use this for frame allocation in stmrewrite for call_assembler
+* use specialized barriers in JIT
+* optimize produced assembler code
+* avoid calling aroundstate.after() for call_release_gil and instead
+  start a normal transaction after the call
+* maybe GUARD_NOT_INEVITABLE after call_may_force, call_assembler
+  which is a small check if we are inevitable and does a transaction_break
+  if we are.
+* look at XXXs for STM everywhere

From noreply at buildbot.pypy.org  Tue Oct 15 11:54:06 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 15 Oct 2013 11:54:06 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: stm blog post update
Message-ID: <20131015095406.8F3C11C011A@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: extradoc
Changeset: r5084:ff9a573f8094
Date: 2013-10-15 11:53 +0200
http://bitbucket.org/pypy/extradoc/changeset/ff9a573f8094/

Log:	stm blog post update

diff --git a/blog/draft/stm-oct2013.rst b/blog/draft/stm-oct2013.rst
--- a/blog/draft/stm-oct2013.rst
+++ b/blog/draft/stm-oct2013.rst
@@ -5,21 +5,27 @@
 
 the sprint in London was a lot of fun and very fruitful. In the last
 update on STM, Armin was working on improving and specializing the 
-automatic barrier placement.
-There is still a lot to do in that area, but that work was merged and
-lowered the overhead of STM over non-STM to around **XXX**. The same
-improvement has still to be done in the JIT.
+automatic barrier placement. There is still a lot to do in that area, 
+but that work is merged now. Specializing and improving barrier placement
+is still to be done for the JIT.
 
 But that is not all. Right after the sprint, we were able to squeeze
 the last obvious bugs in the STM-JIT combination. However, the performance
 was nowhere near to what we want. So until now, we fixed some of the most
 obvious issues. Many come from RPython erring on the side of caution
 and e.g. making a transaction inevitable even if that is not strictly
-necessary, thereby limiting parallelism.
-**XXX any interesting details? transaction breaks maybe? guard counters?**
-There are still many performance issues of various complexity left
+necessary, thereby limiting parallelism. Another problem came from 
+increasing counters everytime a guard fails, which caused transactions
+to conflict on these counter updates. Since these counters do not have
+to be completely accurate, we update them non-transactionally now with
+a chance of small errors.
+
+There are still many such performance issues of various complexity left
 to tackle. So stay tuned or contribute :)
 
+Performance
+-----------
+
 Now, since the JIT is all about performance, we want to at least 
 show you some numbers that are indicative of things to come.
 Our set of STM benchmarks is very small unfortunately 
@@ -61,6 +67,9 @@
 For comparison, disabling the JIT gives 492ms on PyPy-2.1 and 538ms on
 PyPy-STM.
 
+Try it!
+-------
+
 All this can be found in the `PyPy repository on the stmgc-c4
 branch <https://bitbucket.org/pypy/pypy/commits/branch/stmgc-c4>`_.
 Try it for yourself, but keep in mind that this is still experimental
@@ -68,7 +77,10 @@
 
 You can also download a prebuilt binary from here: **XXX**
 
-As a summary, what the numbers tell us is that PyPy-STM is, as expected,
+Summary
+-------
+
+What the numbers tell us is that PyPy-STM is, as expected,
 the only of the three interpreters where multithreading gives a large
 improvement in speed.  What they also tell us is that, obviously, the
 result is not good enough *yet:* it still takes longer on a 8-threaded
@@ -76,3 +88,8 @@
 by now, we are good at promizing speed and delivering it years later.
 It has been two years already since PyPy-STM started, so we're in the
 fast-progressing step right now :-)
+
+
+Cheers
+
+Armin & Remi

From noreply at buildbot.pypy.org  Tue Oct 15 11:54:07 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 15 Oct 2013 11:54:07 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: merge
Message-ID: <20131015095407.D398D1C0203@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: extradoc
Changeset: r5085:33bd3a8dd88d
Date: 2013-10-15 11:53 +0200
http://bitbucket.org/pypy/extradoc/changeset/33bd3a8dd88d/

Log:	merge

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -124,72 +124,106 @@
 Nitty gritty details
 ====================
 
-XXX insert some links where you can read about terms used
+Here are some nitty gritty details for people really interested in
+Garbage Collection.  This was done as a patch to "minimark", our current
+GC, and called "incminimark" for now.  The former is a generational
+stop-the-world GC.  New objects are allocated "young", which means that
+they initially live in the "nursery", a special zone of a few MB of
+memory.  When the nursery is full, a "minor collection" step moves the
+surviving objects out of the nursery.  This can be done quickly (a few
+millisecond) because we only need to walk through the young objects that
+survive --- usually a small fraction of all young objects; and also by
+far not *all* objects that are alive at this point, but only the young
+ones.  However, from time to time this minor collection is followed by a
+"major collection": in that step, we really need to walk all objects to
+classify which ones are still alive and which ones are now dead
+("marking") and free the memory occupied by the dead ones ("sweeping").
+You can read more details here__.
 
-This was done as a patch to "minimark", our current GC, and called
-"incminimark" for now.  The former is a generational stop-the-world GC.
-New objects are allocated "young", i.e. in the nursery, a special zone
-of a few MB of memory.  When it is full, a "minor collection" step moves
-the surviving objects out of the nursery.  This can be done quickly (a
-few millisecond at most) because we only need to walk through the young
-objects that survive --- usually a small fraction of all young objects.
-From time to time, this minor collection is followed by a "major
-collection": in that step, we walk *all* objects to classify which ones
-are still alive and which ones are now dead (*marking*) and free the
-memory occupied by the dead ones (*sweeping*).
+.. __: http://doc.pypy.org/en/latest/garbage_collection.html#minimark-gc
 
 This "major collection" is what gives the long GC pauses.  To fix this
 problem we made the GC incremental: instead of running one complete
-major collection, we split its work into a variable number of pieces
-and run each piece after every minor collection for a while, until there
-are no more pieces.  The pieces are each doing a fraction of marking, or
-a fraction of sweeping.
+major collection, we split its work into a variable number of pieces and
+run each piece after every minor collection for a while, until there are
+no more pieces.  The pieces are each doing a fraction of marking, or a
+fraction of sweeping.  It adds some few milliseconds after each of these
+minor collections, rather than requiring hundreds of milliseconds in one
+go.
 
 The main issue is that splitting the major collections means that the
-main program is actually running between the pieces, and so can change
-the pointers in the objects to point to other objects.  This is not
-a problem for sweeping: dead objects will remain dead whatever the main
-program does.  However, it is a problem for marking.  Let us see why.
+main program is actually running between the pieces, and so it can
+change the pointers in the objects to point to other objects.  This is
+not a problem for sweeping: dead objects will remain dead whatever the
+main program does.  However, it is a problem for marking.  Let us see
+why.
+
+.. __: http://rubini.us/2013/06/22/concurrent-garbage-collection/
+.. __: http://wiki.luajit.org/New-Garbage-Collector/01fd5e5ca4f95d45e0c4b8a98b49f2b656cc23dd
 
 In terms of the incremental GC literature, objects are either "white",
-"gray" or "black".  They start as "white", become "gray" when they are
-found to be alive, and become "black" when they have been fully
-traversed --- at which point the objects that it points to have
-themselves been marked gray, or maybe are already black.  The gray
-objects are the "frontier" between the black objects that we have found
-to be reachable, and the white objects that represent the unknown part
-of the world.  When there are no more gray objects, the process is
-finished: all remaining white objects are unreachable and can be freed
-(by the following sweeping phase).
+"gray" or "black".  This is called *tri-color marking.*  See for example
+this `blog post about Rubinius`__, or this `page about LuaJIT`__.  The
+objects start as "white" at the beginning of marking; become "gray" when
+they are found to be alive; and become "black" when they have been fully
+traversed.  Marking proceeds by scanning grey objects for pointers to
+white objects.  The white objects found are turned grey, and the grey
+objects scanned are turned black.  When there are no more grey objects,
+the marking phase is complete: all remaining white objects are truly
+unreachable and can be freed (by the following sweeping phase).
 
 In this model, the important part is that a black object can never point
 to a white object: if the latter remains white until the end, it will be
 freed, which is incorrect because the black object itself can still be
-reached.
+reached.  How do we ensure that the main program, running in the middle
+of marking, will not try to write a pointer to white object into a black
+object?  This requires a "write barrier", i.e. a piece of code that runs
+every time we set a pointer into an object or array.  This piece of code
+checks if some (hopefully rare) condition is met, and calls a function
+if that is the case.
 
 The trick we used in PyPy is to consider minor collections as part of
 the whole, rather than focus only on major collections.  The existing
-minimark GC had always used a "write barrier" (a piece of code run every time
-you set or get a pointer from an object or array) to do its job, like any
-generational GC.  This write barrier is used to detect when an old
-object (outside the nursery) is modified to point to a young object
-(inside the nursery), which is essential information for minor
+minimark GC had always used a write barrier of its own to do its job,
+like any generational GC.  This existing write barrier is used to detect
+when an old object (outside the nursery) is modified to point to a young
+object (inside the nursery), which is essential information for minor
 collections.  Actually, although this was the goal, the actual write
-barrier code was simpler: it just recorded all old objects into which we
-wrote *any* pointer --- to a young or old object.  It is actually a
-performance improvement, because we don't need to check over and over
-again if the written pointer points to a young object or not.
+barrier code is simpler: it just records all old objects into which we
+write *any* pointer --- to a young or old object.  As we found out over
+time, doing so is not actually slower, and might actually be a
+performance improvement: for example, if the main program does a lot of
+writes into the same old object, we don't need to check over and over
+again if the written pointer points to a young object or not.  We just
+record the old object in some list the first time, and that's it.
 
-This *unmodified* write barrier works for incminimark too.  Imagine that
-we are in the middle of the marking phase, running the main program.
-The write barrier will record all old objects that are being modified.
-Then at the next minor collection, all surviving young objects will be
-moved out of the nursery.  At this point, as we're about to continue
-running the major collection's marking phase, we simply add to the list
-of pending gray objects all the objects that we consider --- both the
-objects listed as "old objects that are being modified", and the objects
-that we just moved out of the nursery.  A fraction of the former list
-are turned back from the black to the gray color.  This technique
+The trick is that this *unmodified* write barrier works for incminimark
+too.  Imagine that we are in the middle of the marking phase, running
+the main program.  The write barrier will record all old objects that
+are being modified.  Then at the next minor collection, all surviving
+young objects will be moved out of the nursery.  At this point, as we're
+about to continue running the major collection's marking phase, we
+simply add to the list of pending gray objects all the objects that we
+just considered --- both the objects listed as "old objects that are
+being modified", and the objects that we just moved out of the nursery.
+A fraction from the former list were black object; so this mean that
+they are turned back from the black to the gray color.  This technique
 implements nicely, if indirectly, what is called a "backward write
-barrier" in the literature: the backwardness is about the color that
-occasionally progresses backward from black to gray.
+barrier" in the literature.  The backwardness is about the color that
+needs to be changed in the opposite of the usual direction "white ->
+gray -> black", thus making more work for the GC.  (This is as opposed
+to "forward write barrier", where we would also detect "black -> white"
+writes but turn the white object gray.)
+
+In summary, I realize that this description is less about how we turned
+minimark into incminimark, and more about how we differ from the
+standard way of making a GC incremental.  What we really had to do to
+make incminimark was to write logic that says "if the major collection
+is in the middle of the marking phase, then add this object to the list
+of gray objects", and put it at a few places throughout minor
+collection.  Then we simply split a major collection into increments,
+doing marking or sweeping of some (relatively arbitrary) number of
+objects before returning.  That's why, after we found that the existing
+write barrier would do, it was not much actual work, and could be done
+without major changes.  For example, not a single line from the JIT
+needed adaptation.  All in all it was relatively painless work. ``:-)``

From noreply at buildbot.pypy.org  Tue Oct 15 12:00:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 15 Oct 2013 12:00:32 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Very long chains or trees of
 objects cause unbounded pauses in CPython too.
Message-ID: <20131015100032.F033D1C0203@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5086:d1b58d4ec41e
Date: 2013-10-15 12:00 +0200
http://bitbucket.org/pypy/extradoc/changeset/d1b58d4ec41e/

Log:	Very long chains or trees of objects cause unbounded pauses in
	CPython too.

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -9,9 +9,12 @@
 Let's start with explaining roughly what GC pauses are. In CPython each
 object has a reference count, which is incremented each time we create
 references and decremented each time we forget them. This means that objects
-(XXX also, very long chains of objects cause unbounded pauses in CPython)
 are freed each time they become unreachable. That is only half of the story
-though. Consider code like this::
+though. First note that when the last reference to a large tree of
+objects goes away, you have a pause: all the objects are freed. Your
+program is not progressing at all during this pause, and this pause's
+duration can be arbitrarily large. This occurs at deterministic times,
+though. But consider code like this::
 
    class A(object):
         pass
@@ -29,8 +32,8 @@
 from the outside. CPython employs a cyclic garbage collector which is used to
 find such cycles. It walks over all objects in memory, starting from some known
 roots, such as ``type`` objects, variables on the stack, etc. This solves the
-problem, but can create noticable GC pauses as the heap becomes large and
-convoluted.
+problem, but can create noticable, undeterministic GC pauses as the heap
+becomes large and convoluted.
 
 PyPy essentially has only the cycle finder - it does not bother with reference
 counting, instead it walks alive objects every now and then (this is a big

From noreply at buildbot.pypy.org  Tue Oct 15 12:09:30 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 15 Oct 2013 12:09:30 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Typos
Message-ID: <20131015100930.A2FB21C011A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5087:4efcbf5b53e0
Date: 2013-10-15 12:09 +0200
http://bitbucket.org/pypy/extradoc/changeset/4efcbf5b53e0/

Log:	Typos

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -54,7 +54,7 @@
 
 Everyone loves benchmarks. We did not measure any significant speed difference
 on our quite extensive benchmark suite on speed.pypy.org. The main
-benchmark that we used for other comparisons was translating `topaz`_
+benchmark that we used for other comparisons was translating the `topaz`_
 ruby interpreter using various versions of PyPy and CPython. The exact
 command was ``python <pypy-checkout>/bin/rpython -O2 --rtype targettopaz.py``.
 Versions:
@@ -68,7 +68,7 @@
 The memory usage of CPython, PyPy with minimark and PyPy with incminimark is
 shown here. Note that this benchmark is quite bad for PyPy in general, the
 memory usage is higher and the amount of time taken is longer. This is due
-to JIT warmup being both memory hungry and inefficient. We'll work on it next.
+to the JIT warmup being both memory hungry and inefficient (see below).
 But first, the new GC is not worst than the old one.
 
 .. image:: memusage.png
@@ -86,8 +86,8 @@
 number of events are not directly comparable since it only shows the cyclic
 collector, not the reference counts. The only comparable thing is the
 amount of long pauses and their duration. In the table below, pause duration
-is sorted into 8 buckets, each meaning "below or equal the threshold".
-The output is generated using `gcanalyze`_ tool.
+is sorted into 8 buckets, each meaning "below that or equal to the threshold".
+The output is generated using the `gcanalyze`_ tool.
 
 .. _`gcanalyze`: https://bitbucket.org/pypy/pypy/src/default/rpython/tool/gcanalyze.py?at=default
 
@@ -120,9 +120,9 @@
 be split among several steps), we did improve the situation quite drastically
 without any actual performance difference.
 
-Note about the benchmark - we know it's a pretty extreme case of a JIT
+Note about the benchmark - we know it's a pretty extreme case of JIT
 warmup, we know we suck on it, we're working on it and we're not afraid of
-showing PyPy is not always the best ;-)
+showing PyPy is not always the best ``;-)``
 
 Nitty gritty details
 ====================

From noreply at buildbot.pypy.org  Tue Oct 15 12:10:44 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 15 Oct 2013 12:10:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: Tweak collect(0), collect(1),
	collect(2).
Message-ID: <20131015101044.CD0041C011A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67391:24146d14cfb0
Date: 2013-10-15 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/24146d14cfb0/

Log:	Tweak collect(0), collect(1), collect(2).

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -653,12 +653,15 @@
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
 
-    def collect(self, gen=1):
-        """Do a minor (gen=0) or full major (gen>0) collection."""
-        if gen > 0:
+    def collect(self, gen=2):
+        """Do a minor (gen=0), start a major (gen=1), or do a full
+        major (gen>=2) collection."""
+        if gen <= 1:
+            self.minor_collection()
+            if gen == 1 or self.gc_state != STATE_SCANNING:
+                self.major_collection_step()
+        else:
             self.minor_and_major_collection()
-        else:
-            self.minor_collection()
 
     def move_nursery_top(self, totalsize):
         size = self.nursery_cleanup

From noreply at buildbot.pypy.org  Tue Oct 15 12:12:12 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 15 Oct 2013 12:12:12 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Capitalization
Message-ID: <20131015101212.98BDC1C011A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5088:20e8cde2f2fa
Date: 2013-10-15 12:11 +0200
http://bitbucket.org/pypy/extradoc/changeset/20e8cde2f2fa/

Log:	Capitalization

diff --git a/blog/draft/stm-oct2013.rst b/blog/draft/stm-oct2013.rst
--- a/blog/draft/stm-oct2013.rst
+++ b/blog/draft/stm-oct2013.rst
@@ -3,7 +3,7 @@
 
 Hi all,
 
-the sprint in London was a lot of fun and very fruitful. In the last
+The sprint in London was a lot of fun and very fruitful. In the last
 update on STM, Armin was working on improving and specializing the 
 automatic barrier placement. There is still a lot to do in that area, 
 but that work is merged now. Specializing and improving barrier placement

From noreply at buildbot.pypy.org  Tue Oct 15 12:16:20 2013
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 15 Oct 2013 12:16:20 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: two typos
Message-ID: <20131015101620.6B0CF1C011A@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: extradoc
Changeset: r5089:517c4014f158
Date: 2013-10-15 12:16 +0200
http://bitbucket.org/pypy/extradoc/changeset/517c4014f158/

Log:	two typos

diff --git a/blog/draft/incremental-gc.rst b/blog/draft/incremental-gc.rst
--- a/blog/draft/incremental-gc.rst
+++ b/blog/draft/incremental-gc.rst
@@ -32,7 +32,7 @@
 from the outside. CPython employs a cyclic garbage collector which is used to
 find such cycles. It walks over all objects in memory, starting from some known
 roots, such as ``type`` objects, variables on the stack, etc. This solves the
-problem, but can create noticable, undeterministic GC pauses as the heap
+problem, but can create noticeable, nondeterministic GC pauses as the heap
 becomes large and convoluted.
 
 PyPy essentially has only the cycle finder - it does not bother with reference

From noreply at buildbot.pypy.org  Tue Oct 15 12:26:39 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 12:26:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: cleanup more imports
Message-ID: <20131015102639.90A831C0205@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67392:8430f7d32663
Date: 2013-10-15 05:11 -0400
http://bitbucket.org/pypy/pypy/changeset/8430f7d32663/

Log:	cleanup more imports

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,10 +1,11 @@
-import core
-from core import *
-import lib
-from lib import *
+from . import core
+from .core import *
+from . import lib
+from .lib import *
 
 from __builtin__ import bool, int, long, float, complex, object, unicode, str
-from core import abs, max, min
+
+from .core import round, abs, max, min
 
 __version__ = '1.7.0'
 
@@ -41,11 +42,9 @@
     return d
 
 
-
 __all__ = ['__version__', 'get_include']
 __all__ += core.__all__
 __all__ += lib.__all__
+
 #import sys
 #sys.modules.setdefault('numpy', sys.modules['numpypy'])
-
-
diff --git a/lib_pypy/numpypy/lib/__init__.py b/lib_pypy/numpypy/lib/__init__.py
--- a/lib_pypy/numpypy/lib/__init__.py
+++ b/lib_pypy/numpypy/lib/__init__.py
@@ -1,11 +1,12 @@
-import function_base
-from function_base import *
-import shape_base
-from shape_base import *
-import twodim_base
-from twodim_base import *
+from __future__ import division, absolute_import, print_function
 
-__all__ = []
+import math
+
+from .function_base import *
+from .shape_base import *
+from .twodim_base import *
+
+__all__ = ['math']
 __all__ += function_base.__all__
 __all__ += shape_base.__all__
 __all__ += twodim_base.__all__

From noreply at buildbot.pypy.org  Tue Oct 15 12:26:40 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 12:26:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: move numpypy get_include to match numpy
Message-ID: <20131015102640.C0D4E1C0205@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67393:a8cd97cc4eb3
Date: 2013-10-15 05:22 -0400
http://bitbucket.org/pypy/pypy/changeset/a8cd97cc4eb3/

Log:	move numpypy get_include to match numpy

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -9,40 +9,7 @@
 
 __version__ = '1.7.0'
 
-import os
-def get_include():
-    """
-    Return the directory that contains the NumPy \\*.h header files.
-
-    Extension modules that need to compile against NumPy should use this
-    function to locate the appropriate include directory.
-
-    Notes
-    -----
-    When using ``distutils``, for example in ``setup.py``.
-    ::
-
-        import numpy as np
-        ...
-        Extension('extension_name', ...
-                include_dirs=[np.get_include()])
-        ...
-
-    """
-    try:
-        import numpy
-    except:
-        # running from pypy source directory
-        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-        return os.path.join(head, '../include')
-    else:
-        # using installed numpy core headers
-        import numpy.core as core
-        d = os.path.join(os.path.dirname(core.__file__), 'include')
-    return d
-
-
-__all__ = ['__version__', 'get_include']
+__all__ = ['__version__']
 __all__ += core.__all__
 __all__ += lib.__all__
 
diff --git a/lib_pypy/numpypy/lib/__init__.py b/lib_pypy/numpypy/lib/__init__.py
--- a/lib_pypy/numpypy/lib/__init__.py
+++ b/lib_pypy/numpypy/lib/__init__.py
@@ -5,8 +5,10 @@
 from .function_base import *
 from .shape_base import *
 from .twodim_base import *
+from .utils import *
 
 __all__ = ['math']
 __all__ += function_base.__all__
 __all__ += shape_base.__all__
 __all__ += twodim_base.__all__
+__all__ += utils.__all__
diff --git a/lib_pypy/numpypy/lib/utils.py b/lib_pypy/numpypy/lib/utils.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/lib/utils.py
@@ -0,0 +1,34 @@
+import os
+
+__all__ = ['get_include']
+
+def get_include():
+    """
+    Return the directory that contains the NumPy \\*.h header files.
+
+    Extension modules that need to compile against NumPy should use this
+    function to locate the appropriate include directory.
+
+    Notes
+    -----
+    When using ``distutils``, for example in ``setup.py``.
+    ::
+
+        import numpy as np
+        ...
+        Extension('extension_name', ...
+                include_dirs=[np.get_include()])
+        ...
+
+    """
+    try:
+        import numpy
+    except:
+        # running from pypy source directory
+        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
+        return os.path.join(head, '../include')
+    else:
+        # using installed numpy core headers
+        import numpy.core as core
+        d = os.path.join(os.path.dirname(core.__file__), 'include')
+    return d
diff --git a/pypy/module/test_lib_pypy/numpypy/test_numpy.py b/pypy/module/test_lib_pypy/numpypy/test_numpy.py
--- a/pypy/module/test_lib_pypy/numpypy/test_numpy.py
+++ b/pypy/module/test_lib_pypy/numpypy/test_numpy.py
@@ -59,10 +59,9 @@
         assert 'numpypy' not in dir(numpypy)
 
     def test_get_include(self):
-        import sys
+        import numpypy, os, sys
+        assert 'get_include' in dir(numpypy)
+        path = numpypy.get_include()
         if not hasattr(sys, 'pypy_translation_info'):
             skip("pypy white-box test")
-        import numpypy, os
-        assert 'get_include' in dir(numpypy)
-        path = numpypy.get_include()
         assert os.path.exists(path + '/numpy/arrayobject.h')

From noreply at buildbot.pypy.org  Tue Oct 15 12:26:41 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 12:26:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: add argwhere, flatnonzero
Message-ID: <20131015102641.EFE841C0205@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67394:2add12c32263
Date: 2013-10-15 06:24 -0400
http://bitbucket.org/pypy/pypy/changeset/2add12c32263/

Log:	add argwhere, flatnonzero

diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -1,12 +1,13 @@
 from __future__ import division, absolute_import, print_function
 
 __all__ = [
-           'newaxis', 'ufunc',
+           'newaxis', 'ufunc', 'argwhere',
            'asarray', 'asanyarray', 'base_repr',
            'array_repr', 'array_str', 'set_string_function',
            'array_equal', 'array_equiv', 'outer', 'vdot', 'identity', 'little_endian',
-           'Inf', 'inf', 'infty', 'Infinity', 'nan', 'NaN', 'False_', 'True_',
-           'seterr',
+           'seterr', 'flatnonzero',
+           'Inf', 'inf', 'infty', 'Infinity',
+           'nan', 'NaN', 'False_', 'True_',
           ]
 
 import sys
@@ -165,6 +166,85 @@
     """
     return array(a, dtype, copy=False, order=order, subok=True)
 
+def argwhere(a):
+    """
+    Find the indices of array elements that are non-zero, grouped by element.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+
+    Returns
+    -------
+    index_array : ndarray
+        Indices of elements that are non-zero. Indices are grouped by element.
+
+    See Also
+    --------
+    where, nonzero
+
+    Notes
+    -----
+    ``np.argwhere(a)`` is the same as ``np.transpose(np.nonzero(a))``.
+
+    The output of ``argwhere`` is not suitable for indexing arrays.
+    For this purpose use ``where(a)`` instead.
+
+    Examples
+    --------
+    >>> x = np.arange(6).reshape(2,3)
+    >>> x
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.argwhere(x>1)
+    array([[0, 2],
+           [1, 0],
+           [1, 1],
+           [1, 2]])
+
+    """
+    return transpose(asanyarray(a).nonzero())
+
+def flatnonzero(a):
+    """
+    Return indices that are non-zero in the flattened version of a.
+
+    This is equivalent to a.ravel().nonzero()[0].
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array.
+
+    Returns
+    -------
+    res : ndarray
+        Output array, containing the indices of the elements of `a.ravel()`
+        that are non-zero.
+
+    See Also
+    --------
+    nonzero : Return the indices of the non-zero elements of the input array.
+    ravel : Return a 1-D array containing the elements of the input array.
+
+    Examples
+    --------
+    >>> x = np.arange(-2, 3)
+    >>> x
+    array([-2, -1,  0,  1,  2])
+    >>> np.flatnonzero(x)
+    array([0, 1, 3, 4])
+
+    Use the indices of the non-zero elements as an index array to extract
+    these elements:
+
+    >>> x.ravel()[np.flatnonzero(x)]
+    array([-2, -1,  1,  2])
+
+    """
+    return a.ravel().nonzero()[0]
+
 def base_repr(number, base=2, padding=0):
     """
     Return a string representation of a number in the given base system.
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
@@ -188,6 +188,23 @@
 
 
 class AppTestNumeric(BaseNumpyAppTest):
+    def test_argwhere(self):
+        import numpypy as np
+        x = np.arange(6).reshape(2,3)
+        a = np.argwhere(x>1)
+        assert np.array_equal(a,
+            [[0, 2],
+            [1, 0],
+            [1, 1],
+            [1, 2]]
+        )
+
+    def test_flatnonzero(self):
+        import numpypy as np
+        x = np.arange(-2, 3)
+        a = np.flatnonzero(x)
+        assert np.array_equal(a, [0, 1, 3, 4])
+
     def test_outer(self):
         from numpypy import array, outer
         a = [1, 2, 3]

From noreply at buildbot.pypy.org  Tue Oct 15 12:48:35 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 15 Oct 2013 12:48:35 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: stm blog post updates
Message-ID: <20131015104835.207AE1C05DF@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: extradoc
Changeset: r5090:989fbc6168ad
Date: 2013-10-15 12:48 +0200
http://bitbucket.org/pypy/extradoc/changeset/989fbc6168ad/

Log:	stm blog post updates

diff --git a/blog/draft/stm-oct2013.rst b/blog/draft/stm-oct2013.rst
--- a/blog/draft/stm-oct2013.rst
+++ b/blog/draft/stm-oct2013.rst
@@ -33,36 +33,39 @@
 not representative of real-world performance. We tried to
 minimize the effect of JIT warm-up in the benchmark results.
 
+The machine these benchmarks were executed on has 4 physical
+cores with Hyper-Threading (8 hardware threads).
+
 
 **Raytracer** from `stm-benchmarks <https://bitbucket.org/Raemi/stm-benchmarks/src>`_:
 Render times in seconds for a 1024x1024 image:
 
-+-------------+----------------------+-------------------+
-| Interpreter | Base time: 1 thread  | 8 threads         |
-+=============+======================+===================+
-| PyPy-2.1    |    2.47              |     2.56          |
-+-------------+----------------------+-------------------+
-| CPython     |    81.1              |     73.4          |
-+-------------+----------------------+-------------------+
-| PyPy-STM    |    50.2              |     10.8          |
-+-------------+----------------------+-------------------+
++-------------+----------------------+---------------------+
+| Interpreter | Base time: 1 thread  | 8 threads (speedup) |
++=============+======================+=====================+
+| PyPy-2.1    |    2.47              |     2.56 (0.96x)    |
++-------------+----------------------+---------------------+
+| CPython     |    81.1              |     73.4 (1.1x)     |
++-------------+----------------------+---------------------+
+| PyPy-STM    |    50.2              |     10.8 (4.6x)     |
++-------------+----------------------+---------------------+
 
-For comparison, disabling the JIT gives 148ms on PyPy-2.1 and 87ms on
+For comparison, disabling the JIT gives 148s on PyPy-2.1 and 87s on
 PyPy-STM (with 8 threads).
 
 **Richards** from `PyPy repository on the stmgc-c4
 branch <https://bitbucket.org/pypy/pypy/commits/branch/stmgc-c4>`_:
 Average time per iteration in milliseconds using 8 threads:
 
-+-------------+----------------------+-------------------+
-| Interpreter | Base time: 1 thread  | 8 threads         |
-+=============+======================+===================+
-| PyPy-2.1    |   15.6               |  15.4             |
-+-------------+----------------------+-------------------+
-| CPython     |   239                |  237              |
-+-------------+----------------------+-------------------+
-| PyPy-STM    |   371                |  116              |
-+-------------+----------------------+-------------------+
++-------------+----------------------+---------------------+
+| Interpreter | Base time: 1 thread  | 8 threads (speedup) |
++=============+======================+=====================+
+| PyPy-2.1    |   15.6               |  15.4 (1.01x)       |
++-------------+----------------------+---------------------+
+| CPython     |   239                |  237 (1.01x)        |
++-------------+----------------------+---------------------+
+| PyPy-STM    |   371                |  116 (3.2x)         |
++-------------+----------------------+---------------------+
 
 For comparison, disabling the JIT gives 492ms on PyPy-2.1 and 538ms on
 PyPy-STM.

From noreply at buildbot.pypy.org  Tue Oct 15 13:01:57 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 15 Oct 2013 13:01:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: test for ravel
Message-ID: <20131015110157.A664B1C0203@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67395:d535d39ec0d7
Date: 2013-10-15 06:54 -0400
http://bitbucket.org/pypy/pypy/changeset/d535d39ec0d7/

Log:	test for ravel

diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -24,6 +24,13 @@
         b[1] = 0
         assert argmin(b) == 0
 
+    def test_ravel(self):
+        import numpypy as np
+        a = np.ravel(np.float64(1))
+        assert np.array_equal(a, [1.])
+        a = np.ravel(np.array([[1, 2, 3], [4, 5, 6]]))
+        assert np.array_equal(a, [1, 2, 3, 4, 5, 6])
+
     def test_shape(self):
         # tests taken from numpy/core/fromnumeric.py docstring
         from numpypy import identity, shape

From noreply at buildbot.pypy.org  Tue Oct 15 13:22:13 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 15 Oct 2013 13:22:13 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: stm blog post fixes
Message-ID: <20131015112213.A52AC1C011A@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: extradoc
Changeset: r5091:a6a790947de2
Date: 2013-10-15 13:21 +0200
http://bitbucket.org/pypy/extradoc/changeset/a6a790947de2/

Log:	stm blog post fixes

diff --git a/blog/draft/stm-oct2013.rst b/blog/draft/stm-oct2013.rst
--- a/blog/draft/stm-oct2013.rst
+++ b/blog/draft/stm-oct2013.rst
@@ -79,6 +79,7 @@
 with a lot of things yet to come.
 
 You can also download a prebuilt binary from here: **XXX**
+(Linux x64 only for now)
 
 Summary
 -------
@@ -88,7 +89,7 @@
 improvement in speed.  What they also tell us is that, obviously, the
 result is not good enough *yet:* it still takes longer on a 8-threaded
 PyPy-STM than on a regular single-threaded PyPy-2.1.  As you should know
-by now, we are good at promizing speed and delivering it years later.
+by now, we are good at promising speed and delivering it years later.
 It has been two years already since PyPy-STM started, so we're in the
 fast-progressing step right now :-)
 

From noreply at buildbot.pypy.org  Tue Oct 15 16:55:36 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 15 Oct 2013 16:55:36 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: draft a numpy update / explanation
 of what happened to numpy.py
Message-ID: <20131015145536.8F98A1C0205@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r5092:80e82984bef1
Date: 2013-10-15 17:55 +0300
http://bitbucket.org/pypy/extradoc/changeset/80e82984bef1/

Log:	draft a numpy update / explanation of what happened to numpy.py

diff --git a/blog/draft/numpy_oct_2013.rst b/blog/draft/numpy_oct_2013.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/numpy_oct_2013.rst
@@ -0,0 +1,56 @@
+NumPy Update
+============
+
+Executive summary: If you wish to try out numpy on pypy, the way to do it is
+changing. ``import numpy`` will no longer work out-of-the-box, you will have to
+install a hacked numpy package.
+
+The way it was
+--------------
+
+PyPy implements a builtin module called _numpypy that exports a native ndarray,
+dtype, ufunc, and some other helpers. The current status of the trunk can be
+viewed on the _`numpy status page`. There was a file numpy.py located in the
+sys.path that wrapped the _numpypy module to look like numpy's extension (compiled
+c) modules. So you could download a pypy.tgz, open it up, and do ``import numpy``
+which would emit a warning, but provide some of the numpy functionality.
+
+What came next
+------------
+
+As we become more and more numpy-compatible, we find ourselves copying files
+from numpy into the pypy source code tree, to provide app-level support. Things
+like numpy.eye, numpy.identity, and all the module level funcitons that simply
+create an ndarray and then call the corresponding method on the ndarray.
+
+Since that seems repetative and prone to bitrot, we began a _`fork of numpy`
+to allow mortals to run ``setup.py install`` and get much more of numpy working.
+
+What went wrong, choices, and the decision
+---------------------------------
+
+The numpy.py file masks the numpy package installed in site-packages since it
+appears first on the search path. So we could have hacked at it to import numpy
+if such a package exists and to export it anew. But the better choice was
+made - to remove it entirely.
+
+What the future holds
+--------------------
+
+PyPy is making great progress, the engine is getting better and better, and
+interfacing to packages is getting easier. Currently, using the pypy-hack 
+branch of  the _`fork of numpy`
+and another _`fork of matplotlib`, you can create plots and save them to files.
+These extensions use the immature and slow c api (cpyext) in PyPy. We would
+prefer to use the jit-friendly cffi for extension modules, progress has been
+made writing a cffi interface for wxWidgets in a GSOC, but _`that` is a subject
+for another post. We will soon be uploading the PyPy compatible numpy package
+to _`PyPi`, and would love to get more people involved in hacking, testing,
+and benchmarking numpy on PyPy.
+
+Matti (mattip), and the PyPy team
+
+.. _`numpy status page`: http://buildbot.pypy.org/numpy-status/latest.html
+.. _`fork of numpy`: https://github.com/mattip/numpy
+.. _`fork of matplotlib`: https://github.com/mattip/matplotlib
+.. _`that`: http://waedt.blogspot.co.il

From noreply at buildbot.pypy.org  Tue Oct 15 17:23:36 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 15 Oct 2013 17:23:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: A fix for what turns out to be a
 non-issue, a bit by chance
Message-ID: <20131015152336.93A5C1C1352@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67396:af629245abef
Date: 2013-10-15 17:22 +0200
http://bitbucket.org/pypy/pypy/changeset/af629245abef/

Log:	A fix for what turns out to be a non-issue, a bit by chance

diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -52,10 +52,12 @@
         self.setup_descrs(asm.descrs)
         self.metainterp_sd = metainterp_sd
         self.num_interpreters = 0
-        self._cleanup_()
+        self.blackholeinterps = []
 
     def _cleanup_(self):
-        self.blackholeinterps = []
+        # XXX don't assign a different list to blackholeinterp here,
+        # it confuses the annotator a lot
+        del self.blackholeinterps[:]
 
     def setup_insns(self, insns):
         assert len(insns) <= 256, "too many instructions!"

From noreply at buildbot.pypy.org  Tue Oct 15 19:43:16 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 15 Oct 2013 19:43:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: test, implement ndarray.flat = val
Message-ID: <20131015174316.61FB51C0203@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r67397:eff24d19da2b
Date: 2013-10-15 19:18 +0300
http://bitbucket.org/pypy/pypy/changeset/eff24d19da2b/

Log:	test, implement ndarray.flat = val

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -399,6 +399,10 @@
     def descr_repeat(self, space, repeats, w_axis=None):
         return repeat(space, self, repeats, w_axis)
 
+    def descr_set_flatiter(self, space, w_obj):
+        arr = convert_to_array(space, w_obj)
+        loop.flatiter_setitem(space, self, arr, 0, 1, self.get_size())
+
     def descr_get_flatiter(self, space):
         return space.wrap(W_FlatIterator(self))
 
@@ -1130,7 +1134,8 @@
     repeat = interp2app(W_NDimArray.descr_repeat),
     swapaxes = interp2app(W_NDimArray.descr_swapaxes),
     nonzero = interp2app(W_NDimArray.descr_nonzero),
-    flat = GetSetProperty(W_NDimArray.descr_get_flatiter),
+    flat = GetSetProperty(W_NDimArray.descr_get_flatiter,
+                          W_NDimArray.descr_set_flatiter),
     item = interp2app(W_NDimArray.descr_item),
     real = GetSetProperty(W_NDimArray.descr_get_real,
                           W_NDimArray.descr_set_real),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2273,6 +2273,24 @@
         assert len(arange(10)[:2].flat) == 2
         assert len((arange(2) + arange(2)).flat) == 2
 
+    def test_flatiter_setter(self):
+        from numpypy import arange, array
+        a = arange(24).reshape(2, 3, 4)
+        a.flat = [4, 5]
+        assert (a.flatten() == [4, 5]*12).all()
+        a.flat = [[4, 5, 6, 7, 8], [4, 5, 6, 7, 8]]
+        assert (a.flatten() == ([4, 5, 6, 7, 8]*5)[:24]).all()
+        exc = raises(ValueError, 'a.flat = [[4, 5, 6, 7, 8], [4, 5, 6]]')
+        assert str(exc.value).find("sequence") > 0
+        b = a[::-1, :, ::-1]
+        b.flat = range(24)
+        assert (a.flatten() == [15, 14 ,13, 12, 19, 18, 17, 16, 23, 22,
+                                21, 20, 3, 2, 1, 0, 7, 6, 5, 4,
+                                11, 10, 9, 8]).all()
+        c = array(['abc'] * 10).reshape(2, 5)
+        c.flat = ['defgh', 'ijklmnop']
+        assert (c.flatten() == ['def', 'ijk']*5).all()
+
     def test_slice_copy(self):
         from numpypy import zeros
         a = zeros((10, 10))

From noreply at buildbot.pypy.org  Tue Oct 15 19:43:17 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 15 Oct 2013 19:43:17 +0200 (CEST)
Subject: [pypy-commit] pypy cpyext-int: a failing test
Message-ID: <20131015174317.BCC4B1C0205@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-int
Changeset: r67398:601ae6115bb6
Date: 2013-10-15 20:38 +0300
http://bitbucket.org/pypy/pypy/changeset/601ae6115bb6/

Log:	a failing test

diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -286,3 +286,21 @@
         arr = mod.test_FromObject()
         dt = mod.test_DescrFromType(11)
         assert dt.num == 11
+
+    def test_int_cast(self):
+        mod = self.import_extension('foo', [
+                #prove it works for ints
+                ("test_int", "METH_NOARGS",
+                """
+                PyIntObject * obj = PyInt_FromLong(42);
+                if ( PyInt_Check(obj))
+                    return obj;
+                PyObject * val = PyInt_FromLong(obj->ob_ival);
+                Py_DECREF(obj);
+                return val;
+                """
+                ),
+                ], prologue='#include <numpy/arrayobject.h>')
+        i = mod.test_int()
+        assert isinstance(i, int)
+        assert i == 42

From noreply at buildbot.pypy.org  Tue Oct 15 19:43:18 2013
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 15 Oct 2013 19:43:18 +0200 (CEST)
Subject: [pypy-commit] pypy cpyext-int: fix test by returning a PyIntObject
	where needed
Message-ID: <20131015174318.DD5881C0203@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-int
Changeset: r67399:2027437ec5a6
Date: 2013-10-15 20:41 +0300
http://bitbucket.org/pypy/pypy/changeset/2027437ec5a6/

Log:	fix test by returning a PyIntObject where needed

diff --git a/pypy/module/cpyext/intobject.py b/pypy/module/cpyext/intobject.py
--- a/pypy/module/cpyext/intobject.py
+++ b/pypy/module/cpyext/intobject.py
@@ -41,10 +41,10 @@
     as defined in the system header files)."""
     return sys.maxint
 
- at cpython_api([lltype.Signed], PyObject)
+ at cpython_api([lltype.Signed], PyIntObject)
 def PyInt_FromLong(space, ival):
     """Create a new integer object with a value of ival.
-    
+
     """
     return space.wrap(ival)
 
@@ -117,7 +117,7 @@
 
 LONG_MAX = int(LONG_TEST - 1)
 
- at cpython_api([rffi.SIZE_T], PyObject)
+ at cpython_api([rffi.SIZE_T], PyIntObject)
 def PyInt_FromSize_t(space, ival):
     """Create a new integer object with a value of ival. If the value exceeds
     LONG_MAX, a long integer object is returned.
@@ -126,7 +126,7 @@
         return space.wrap(intmask(ival))
     return space.wrap(ival)
 
- at cpython_api([Py_ssize_t], PyObject)
+ at cpython_api([Py_ssize_t], PyIntObject)
 def PyInt_FromSsize_t(space, ival):
     """Create a new integer object with a value of ival. If the value is larger
     than LONG_MAX or smaller than LONG_MIN, a long integer object is
@@ -134,7 +134,7 @@
     """
     return space.wrap(ival)
 
- at cpython_api([CONST_STRING, rffi.CCHARPP, rffi.INT_real], PyObject)
+ at cpython_api([CONST_STRING, rffi.CCHARPP, rffi.INT_real], PyIntObject)
 def PyInt_FromString(space, str, pend, base):
     """Return a new PyIntObject or PyLongObject based on the string
     value in str, which is interpreted according to the radix in base.  If

From noreply at buildbot.pypy.org  Wed Oct 16 01:22:35 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 01:22:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: implement and test ndarray.trace()
Message-ID: <20131015232235.D387F1C0205@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67400:3fd593fe30bf
Date: 2013-10-15 19:09 -0400
http://bitbucket.org/pypy/pypy/changeset/3fd593fe30bf/

Log:	implement and test ndarray.trace()

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -550,6 +550,12 @@
         return interp_arrayops.diagonal(space, self.implementation, offset,
                                         axis1, axis2)
 
+    @unwrap_spec(offset=int, axis1=int, axis2=int)
+    def descr_trace(self, space, offset=0, axis1=0, axis2=1,
+                    w_dtype=None, w_out=None):
+        diag = self.descr_diagonal(space, offset, axis1, axis2)
+        return diag.descr_sum(space, w_axis=space.wrap(-1), w_dtype=w_dtype, w_out=w_out)
+
     def descr_dump(self, space, w_file):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "dump not implemented yet"))
@@ -653,11 +659,6 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "tofile not implemented yet"))
 
-    def descr_trace(self, space, w_offset=0, w_axis1=0, w_axis2=1,
-                    w_dtype=None, w_out=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "trace not implemented yet"))
-
     def descr_view(self, space, w_dtype=None, w_type=None) :
         if not w_type and w_dtype:
             try:
@@ -1153,6 +1154,7 @@
     round    = interp2app(W_NDimArray.descr_round),
     data     = GetSetProperty(W_NDimArray.descr_get_data),
     diagonal = interp2app(W_NDimArray.descr_diagonal),
+    trace = interp2app(W_NDimArray.descr_trace),
     view = interp2app(W_NDimArray.descr_view),
 
     ctypes = GetSetProperty(W_NDimArray.descr_get_ctypes), # XXX unimplemented
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1465,6 +1465,14 @@
         assert a[3].imag == -10
         assert a[2].imag == -5
 
+    def test_trace(self):
+        import numpypy as np
+        assert np.trace(np.eye(3)) == 3.0
+        a = np.arange(8).reshape((2,2,2))
+        assert np.array_equal(np.trace(a), [6, 8])
+        a = np.arange(24).reshape((2,2,2,3))
+        assert np.trace(a).shape == (2, 3)
+
     def test_view(self):
         from numpypy import array, int8, int16, dtype
         x = array((1, 2), dtype=int8)

From noreply at buildbot.pypy.org  Wed Oct 16 01:22:37 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 01:22:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: test/fix position of dtype/out args for
 ndarray reduce ufuncs
Message-ID: <20131015232237.1358A1C02C2@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67401:cca686ae3c69
Date: 2013-10-15 19:21 -0400
http://bitbucket.org/pypy/pypy/changeset/cca686ae3c69/

Log:	test/fix position of dtype/out args for ndarray reduce ufuncs

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -846,7 +846,7 @@
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False,
                            cumultative=False):
-        def impl(self, space, w_axis=None, w_out=None, w_dtype=None):
+        def impl(self, space, w_axis=None, w_dtype=None, w_out=None):
             if space.is_none(w_out):
                 out = None
             elif not isinstance(w_out, W_NDimArray):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2156,6 +2156,7 @@
         c = b + b
         assert c.sum() == (6 + 8 + 10 + 12) * 2
         assert isinstance(c.sum(dtype='f8'), float)
+        assert isinstance(c.sum(None, 'f8'), float)
 
     def test_transpose(self):
         from numpypy import array

From noreply at buildbot.pypy.org  Wed Oct 16 01:24:34 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 16 Oct 2013 01:24:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: Document this branch
Message-ID: <20131015232434.E4E001C0205@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67402:b451615b3ded
Date: 2013-10-15 16:23 -0700
http://bitbucket.org/pypy/pypy/changeset/b451615b3ded/

Log:	Document this branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -109,4 +109,5 @@
 .. branch: file-support-in-rpython
 make open() and friends rpython
 
-
+.. branch: incremental-gc
+Added the new incminimark GC which performs GC in incremental steps
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -9,20 +9,6 @@
 from pypy.interpreter.typedef import TypeDef
 
 
-##import sys
-##def debug(msg, n):
-##    return
-##    tb = []
-##    try:
-##        for i in range(1, 8):
-##            tb.append(sys._getframe(i).f_code.co_name)
-##    except:
-##        pass
-##    tb = ' '.join(tb)
-##    msg = '| %6d | %d %s | %s\n' % (rthread.get_ident(), n, msg, tb)
-##    sys.stderr.write(msg)
-
-
 class Lock(W_Root):
     "A box around an interp-level lock object."
 

From noreply at buildbot.pypy.org  Wed Oct 16 01:24:36 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 16 Oct 2013 01:24:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20131015232436.4122C1C0205@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67403:49a8eaba23b0
Date: 2013-10-15 16:24 -0700
http://bitbucket.org/pypy/pypy/changeset/49a8eaba23b0/

Log:	merged upstream

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -550,6 +550,12 @@
         return interp_arrayops.diagonal(space, self.implementation, offset,
                                         axis1, axis2)
 
+    @unwrap_spec(offset=int, axis1=int, axis2=int)
+    def descr_trace(self, space, offset=0, axis1=0, axis2=1,
+                    w_dtype=None, w_out=None):
+        diag = self.descr_diagonal(space, offset, axis1, axis2)
+        return diag.descr_sum(space, w_axis=space.wrap(-1), w_dtype=w_dtype, w_out=w_out)
+
     def descr_dump(self, space, w_file):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "dump not implemented yet"))
@@ -653,11 +659,6 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "tofile not implemented yet"))
 
-    def descr_trace(self, space, w_offset=0, w_axis1=0, w_axis2=1,
-                    w_dtype=None, w_out=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "trace not implemented yet"))
-
     def descr_view(self, space, w_dtype=None, w_type=None) :
         if not w_type and w_dtype:
             try:
@@ -845,7 +846,7 @@
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False,
                            cumultative=False):
-        def impl(self, space, w_axis=None, w_out=None, w_dtype=None):
+        def impl(self, space, w_axis=None, w_dtype=None, w_out=None):
             if space.is_none(w_out):
                 out = None
             elif not isinstance(w_out, W_NDimArray):
@@ -1153,6 +1154,7 @@
     round    = interp2app(W_NDimArray.descr_round),
     data     = GetSetProperty(W_NDimArray.descr_get_data),
     diagonal = interp2app(W_NDimArray.descr_diagonal),
+    trace = interp2app(W_NDimArray.descr_trace),
     view = interp2app(W_NDimArray.descr_view),
 
     ctypes = GetSetProperty(W_NDimArray.descr_get_ctypes), # XXX unimplemented
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1465,6 +1465,14 @@
         assert a[3].imag == -10
         assert a[2].imag == -5
 
+    def test_trace(self):
+        import numpypy as np
+        assert np.trace(np.eye(3)) == 3.0
+        a = np.arange(8).reshape((2,2,2))
+        assert np.array_equal(np.trace(a), [6, 8])
+        a = np.arange(24).reshape((2,2,2,3))
+        assert np.trace(a).shape == (2, 3)
+
     def test_view(self):
         from numpypy import array, int8, int16, dtype
         x = array((1, 2), dtype=int8)
@@ -2148,6 +2156,7 @@
         c = b + b
         assert c.sum() == (6 + 8 + 10 + 12) * 2
         assert isinstance(c.sum(dtype='f8'), float)
+        assert isinstance(c.sum(None, 'f8'), float)
 
     def test_transpose(self):
         from numpypy import array

From noreply at buildbot.pypy.org  Wed Oct 16 02:51:37 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 16 Oct 2013 02:51:37 +0200 (CEST)
Subject: [pypy-commit] pypy python-loop-unroll: merged default in
Message-ID: <20131016005137.D9F341C0203@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: python-loop-unroll
Changeset: r67404:8b906858a45e
Date: 2013-10-15 17:50 -0700
http://bitbucket.org/pypy/pypy/changeset/8b906858a45e/

Log:	merged default in

diff too long, truncating to 2000 out of 86015 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -297,30 +297,6 @@
 under the Python Software License of which you can find a copy here:
 http://www.python.org/doc/Copyright.html 
 
-License for 'rpython/translator/jvm/src/jna.jar'
-=============================================
-
-The file 'rpyhton/translator/jvm/src/jna.jar' is licensed under the GNU
-Lesser General Public License of which you can find a copy here:
-http://www.gnu.org/licenses/lgpl.html
-
-License for 'rpython/translator/jvm/src/jasmin.jar'
-================================================
-
-The file 'rpyhton/translator/jvm/src/jasmin.jar' is copyright (c) 1996-2004 Jon Meyer
-and distributed with permission.  The use of Jasmin by PyPy does not imply
-that PyPy is endorsed by Jon Meyer nor any of Jasmin's contributors.  Furthermore,
-the following disclaimer applies to Jasmin:
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
-WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
-TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
-ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
 License for 'pypy/module/unicodedata/'
 ======================================
 
diff --git a/dotviewer/drawgraph.py b/dotviewer/drawgraph.py
--- a/dotviewer/drawgraph.py
+++ b/dotviewer/drawgraph.py
@@ -22,6 +22,7 @@
     'yellow': (255,255,0),
     }
 re_nonword=re.compile(r'([^0-9a-zA-Z_.]+)')
+re_linewidth=re.compile(r'setlinewidth\((\d+(\.\d*)?|\.\d+)\)')
 
 def combine(color1, color2, alpha):
     r1, g1, b1 = color1
@@ -138,6 +139,13 @@
             self.yl = float(yl)
             rest = rest[3:]
         self.style, self.color = rest
+        linematch = re_linewidth.match(self.style)
+        if linematch:
+            num = linematch.group(1)
+            self.linewidth = int(round(float(num)))
+            self.style = self.style[linematch.end(0):]
+        else:
+            self.linewidth = 1
         self.highlight = False
         self.cachedbezierpoints = None
         self.cachedarrowhead = None
@@ -520,8 +528,8 @@
                 fgcolor = highlight_color(fgcolor)
             points = [self.map(*xy) for xy in edge.bezierpoints()]
 
-            def drawedgebody(points=points, fgcolor=fgcolor):
-                pygame.draw.lines(self.screen, fgcolor, False, points)
+            def drawedgebody(points=points, fgcolor=fgcolor, width=edge.linewidth):
+                pygame.draw.lines(self.screen, fgcolor, False, points, width)
             edgebodycmd.append(drawedgebody)
 
             points = [self.map(*xy) for xy in edge.arrowhead()]
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -152,7 +152,8 @@
         try:
             plaincontent = dot2plain_graphviz(content, contenttype)
         except PlainParseError, e:
-            print e
-            # failed, retry via codespeak
-            plaincontent = dot2plain_codespeak(content, contenttype)
+            raise
+            ##print e
+            ### failed, retry via codespeak
+            ##plaincontent = dot2plain_codespeak(content, contenttype)
     return list(parse_plain(graph_id, plaincontent, links, fixedfont))
diff --git a/dotviewer/test/test_interactive.py b/dotviewer/test/test_interactive.py
--- a/dotviewer/test/test_interactive.py
+++ b/dotviewer/test/test_interactive.py
@@ -34,6 +34,23 @@
 }
 '''
 
+SOURCE2=r'''digraph f {
+  a; d; e; f; g; h; i; j; k; l;
+  a -> d [penwidth=1, style="setlinewidth(1)"];
+  d -> e [penwidth=2, style="setlinewidth(2)"];
+  e -> f [penwidth=4, style="setlinewidth(4)"];
+  f -> g [penwidth=8, style="setlinewidth(8)"];
+  g -> h [penwidth=16, style="setlinewidth(16)"];
+  h -> i [penwidth=32, style="setlinewidth(32)"];
+  i -> j [penwidth=64, style="setlinewidth(64)"];
+  j -> k [penwidth=128, style="setlinewidth(128)"];
+  k -> l [penwidth=256, style="setlinewidth(256)"];
+}'''
+
+
+
+
+
 def setup_module(mod):
     if not option.pygame:
         py.test.skip("--pygame not enabled")
@@ -161,3 +178,10 @@
     page = MyPage(str(dotfile))
     page.fixedfont = True
     graphclient.display_page(page)
+
+def test_linewidth():
+    udir.join("graph2.dot").write(SOURCE2)
+    from dotviewer import graphpage, graphclient
+    dotfile = udir.join('graph2.dot')
+    page = graphpage.DotFileGraphPage(str(dotfile))
+    graphclient.display_page(page)
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -1780,7 +1780,19 @@
             # error if this argument is not allowed with other previously
             # seen arguments, assuming that actions that use the default
             # value don't really count as "present"
-            if argument_values is not action.default:
+
+            # XXX PyPy bug-to-bug compatibility: "is" on primitive types
+            # is not consistent in CPython.  We'll assume it is close
+            # enough for ints (which is true only for "small ints"), but
+            # for floats and longs and complexes we'll go for the option
+            # of forcing "is" to say False, like it usually does on
+            # CPython.  A fix is pending on CPython trunk
+            # (http://bugs.python.org/issue18943) but that might change
+            # the details of the semantics and so not be applied to 2.7.
+            # See the line AA below.
+
+            if (argument_values is not action.default or
+                    type(argument_values) in (float, long, complex)):  # AA
                 seen_non_default_actions.add(action)
                 for conflict_action in action_conflicts.get(action, []):
                     if conflict_action in seen_non_default_actions:
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,6 +12,7 @@
 
 import sys
 import os
+import shlex
 
 from distutils.errors import DistutilsPlatformError
 
@@ -65,11 +66,6 @@
     g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
-    g['OPT'] = ""
-    g['CFLAGS'] = ""
-    g['CPPFLAGS'] = ""
-    g['CCSHARED'] = '-shared -O2 -fPIC -Wimplicit'
-    g['LDSHARED'] = g['CC'] + ' -shared'
 
     global _config_vars
     _config_vars = g
@@ -127,34 +123,21 @@
     optional C speedup components.
     """
     if compiler.compiler_type == "unix":
-        cc, opt, cflags, ccshared, ldshared = get_config_vars(
-            'CC', 'OPT', 'CFLAGS', 'CCSHARED', 'LDSHARED')
-
+        compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
-
-        if 'LDSHARED' in os.environ:
-            ldshared = os.environ['LDSHARED']
-        if 'CPP' in os.environ:
-            cpp = os.environ['CPP']
-        else:
-            cpp = cc + " -E"           # not always
-        if 'LDFLAGS' in os.environ:
-            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
-        if 'CFLAGS' in os.environ:
-            cflags = opt + ' ' + os.environ['CFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CFLAGS']
-        if 'CPPFLAGS' in os.environ:
-            cpp = cpp + ' ' + os.environ['CPPFLAGS']
-            cflags = cflags + ' ' + os.environ['CPPFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
-
-        cc_cmd = cc + ' ' + cflags
-
-        compiler.set_executables(
-            preprocessor=cpp,
-            compiler=cc_cmd,
-            compiler_so=cc_cmd + ' ' + ccshared,
-            linker_so=ldshared)
+        if "CPPFLAGS" in os.environ:
+            cppflags = shlex.split(os.environ["CPPFLAGS"])
+            compiler.compiler.extend(cppflags)
+            compiler.compiler_so.extend(cppflags)
+            compiler.linker_so.extend(cppflags)
+        if "CFLAGS" in os.environ:
+            cflags = shlex.split(os.environ["CFLAGS"])
+            compiler.compiler.extend(cflags)
+            compiler.compiler_so.extend(cflags)
+            compiler.linker_so.extend(cflags)
+        if "LDFLAGS" in os.environ:
+            ldflags = shlex.split(os.environ["LDFLAGS"])
+            compiler.linker_so.extend(ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -165,6 +165,8 @@
     # All _delegate_methods must also be initialized here.
     send = recv = recv_into = sendto = recvfrom = recvfrom_into = _dummy
     __getattr__ = _dummy
+    def _drop(self):
+        pass
 
 # Wrapper around platform socket objects. This implements
 # a platform-independent dup() functionality. The
@@ -179,12 +181,21 @@
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
-        elif _type(_sock) is _realsocket:
+        else:
+            # PyPy note about refcounting: implemented with _reuse()/_drop()
+            # on the class '_socket.socket'.  Python 3 did it differently
+            # with a reference counter on this class 'socket._socketobject'
+            # instead, but it is a less compatible change.
+            
+            # Note that a few libraries (like eventlet) poke at the
+            # private implementation of socket.py, passing custom
+            # objects to _socketobject().  These libraries need the
+            # following fix for use on PyPy: the custom objects need
+            # methods _reuse() and _drop() that maintains an explicit
+            # reference counter, starting at 0.  When it drops back to
+            # zero, close() must be called.
             _sock._reuse()
-        # PyPy note about refcounting: implemented with _reuse()/_drop()
-        # on the class '_socket.socket'.  Python 3 did it differently
-        # with a reference counter on this class 'socket._socketobject'
-        # instead, but it is a less compatible change (breaks eventlet).
+
         self._sock = _sock
 
     def send(self, data, flags=0):
@@ -216,9 +227,8 @@
 
     def close(self):
         s = self._sock
-        if type(s) is _realsocket:
-            s._drop()
         self._sock = _closedsocket()
+        s._drop()
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
@@ -280,8 +290,14 @@
                  "_close"]
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
-        if type(sock) is _realsocket:
-            sock._reuse()
+        # Note that a few libraries (like eventlet) poke at the
+        # private implementation of socket.py, passing custom
+        # objects to _fileobject().  These libraries need the
+        # following fix for use on PyPy: the custom objects need
+        # methods _reuse() and _drop() that maintains an explicit
+        # reference counter, starting at 0.  When it drops back to
+        # zero, close() must be called.
+        sock._reuse()
         self._sock = sock
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
@@ -317,11 +333,11 @@
                 self.flush()
         finally:
             s = self._sock
-            if type(s) is _realsocket:
+            self._sock = None
+            if s is not None:
                 s._drop()
-            if self._close:
-                self._sock.close()
-            self._sock = None
+                if self._close:
+                    s.close()
 
     def __del__(self):
         try:
diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py
--- a/lib-python/2.7/ssl.py
+++ b/lib-python/2.7/ssl.py
@@ -110,6 +110,12 @@
                  suppress_ragged_eofs=True, ciphers=None):
         socket.__init__(self, _sock=sock._sock)
 
+        # "close" the original socket: it is not usable any more.
+        # this only calls _drop(), which should not actually call
+        # the operating system's close() because the reference
+        # counter is greater than 1 (we hold one too).
+        sock.close()
+
         if ciphers is None and ssl_version != _SSLv2_IF_EXISTS:
             ciphers = _DEFAULT_CIPHERS
 
@@ -352,11 +358,19 @@
         works with the SSL connection.  Just use the code
         from the socket module."""
 
-        self._makefile_refs += 1
         # close=True so as to decrement the reference count when done with
         # the file-like object.
         return _fileobject(self, mode, bufsize, close=True)
 
+    def _reuse(self):
+        self._makefile_refs += 1
+
+    def _drop(self):
+        if self._makefile_refs < 1:
+            self.close()
+        else:
+            self._makefile_refs -= 1
+
 
 
 def wrap_socket(sock, keyfile=None, certfile=None,
diff --git a/lib-python/2.7/test/keycert.pem b/lib-python/2.7/test/keycert.pem
--- a/lib-python/2.7/test/keycert.pem
+++ b/lib-python/2.7/test/keycert.pem
@@ -1,32 +1,31 @@
------BEGIN RSA PRIVATE KEY-----
-MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L
-opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH
-fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB
-AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU
-D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA
-IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM
-oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0
-ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/
-loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j
-oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA
-z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq
-ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV
-q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU=
------END RSA PRIVATE KEY-----
+-----BEGIN PRIVATE KEY-----
+MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm
+LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0
+ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP
+USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt
+CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq
+SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK
+UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y
+BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ
+ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5
+oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik
+eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F
+0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS
+x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/
+SPIXQuT8RMPDVNQ=
+-----END PRIVATE KEY-----
 -----BEGIN CERTIFICATE-----
-MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD
-VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x
-IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT
-U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1
-NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl
-bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m
-dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj
-aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh
-m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8
-M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn
-fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC
-AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb
-08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx
-CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/
-iHkC6gGdBJhogs4=
+MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw
+MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH
+Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k
+YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
+gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7
+6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt
+pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw
+FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd
+BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G
+lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1
+CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX
 -----END CERTIFICATE-----
diff --git a/lib-python/2.7/test/sha256.pem b/lib-python/2.7/test/sha256.pem
--- a/lib-python/2.7/test/sha256.pem
+++ b/lib-python/2.7/test/sha256.pem
@@ -1,129 +1,128 @@
 # Certificate chain for https://sha256.tbs-internet.com
- 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=sha-256 production/CN=sha256.tbs-internet.com
-   i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC
+ 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=Certificats TBS X509/CN=ecom.tbs-x509.com
+   i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business
 -----BEGIN CERTIFICATE-----
-MIIGXTCCBUWgAwIBAgIRAMmag+ygSAdxZsbyzYjhuW0wDQYJKoZIhvcNAQELBQAw
-gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl
+MIIGTjCCBTagAwIBAgIQOh3d9dNDPq1cSdJmEiMpqDANBgkqhkiG9w0BAQUFADCB
+yTELMAkGA1UEBhMCRlIxETAPBgNVBAgTCENhbHZhZG9zMQ0wCwYDVQQHEwRDYWVu
+MRUwEwYDVQQKEwxUQlMgSU5URVJORVQxSDBGBgNVBAsTP1Rlcm1zIGFuZCBDb25k
+aXRpb25zOiBodHRwOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0EvcmVwb3NpdG9y
+eTEYMBYGA1UECxMPVEJTIElOVEVSTkVUIENBMR0wGwYDVQQDExRUQlMgWDUwOSBD
+QSBidXNpbmVzczAeFw0xMTAxMjUwMDAwMDBaFw0xMzAyMDUyMzU5NTlaMIHHMQsw
+CQYDVQQGEwJGUjEOMAwGA1UEERMFMTQwMDAxETAPBgNVBAgTCENhbHZhZG9zMQ0w
+CwYDVQQHEwRDQUVOMRswGQYDVQQJExIyMiBydWUgZGUgQnJldGFnbmUxFTATBgNV
+BAoTDFRCUyBJTlRFUk5FVDEXMBUGA1UECxMOMDAwMiA0NDA0NDM4MTAxHTAbBgNV
+BAsTFENlcnRpZmljYXRzIFRCUyBYNTA5MRowGAYDVQQDExFlY29tLnRicy14NTA5
+LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKRrlHUnJ++1lpcg
+jtYco7cdmRe+EEfTmwPfCdfV3G1QfsTSvY6FfMpm/83pqHfT+4ANwr18wD9ZrAEN
+G16mf9VdCGK12+TP7DmqeZyGIqlFFoahQnmb8EarvE43/1UeQ2CV9XmzwZvpqeli
+LfXsFonawrY3H6ZnMwS64St61Z+9gdyuZ/RbsoZBbT5KUjDEG844QRU4OT1IGeEI
+eY5NM5RNIh6ZNhVtqeeCxMS7afONkHQrOco73RdSTRck/Hj96Ofl3MHNHryr+AMK
+DGFk1kLCZGpPdXtkxXvaDeQoiYDlil26CWc+YK6xyDPMdsWvoG14ZLyCpzMXA7/7
+4YAQRH0CAwEAAaOCAjAwggIsMB8GA1UdIwQYMBaAFBoJBMz5CY+7HqDO1KQUf0vV
+I1jNMB0GA1UdDgQWBBQgOU8HsWzbmD4WZP5Wtdw7jca2WDAOBgNVHQ8BAf8EBAMC
+BaAwDAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw
+TAYDVR0gBEUwQzBBBgsrBgEEAYDlNwIBATAyMDAGCCsGAQUFBwIBFiRodHRwczov
+L3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL0NQUzEwdwYDVR0fBHAwbjA3oDWgM4Yx
+aHR0cDovL2NybC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNy
+bDAzoDGgL4YtaHR0cDovL2NybC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5l
+c3MuY3JsMIGwBggrBgEFBQcBAQSBozCBoDA9BggrBgEFBQcwAoYxaHR0cDovL2Ny
+dC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNydDA5BggrBgEF
+BQcwAoYtaHR0cDovL2NydC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5lc3Mu
+Y3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wMwYDVR0R
+BCwwKoIRZWNvbS50YnMteDUwOS5jb22CFXd3dy5lY29tLnRicy14NTA5LmNvbTAN
+BgkqhkiG9w0BAQUFAAOCAQEArT4NHfbY87bGAw8lPV4DmHlmuDuVp/y7ltO3Ynse
+3Rz8RxW2AzuO0Oy2F0Cu4yWKtMyEyMXyHqWtae7ElRbdTu5w5GwVBLJHClCzC8S9
+SpgMMQTx3Rgn8vjkHuU9VZQlulZyiPK7yunjc7c310S9FRZ7XxOwf8Nnx4WnB+No
+WrfApzhhQl31w+RyrNxZe58hCfDDHmevRvwLjQ785ZoQXJDj2j3qAD4aI2yB8lB5
+oaE1jlCJzC7Kmz/Y9jzfmv/zAs1LQTm9ktevv4BTUFaGjv9jxnQ1xnS862ZiouLW
+zZYIlYPf4F6JjXGiIQgQRglILUfq3ftJd9/ok9W9ZF8h8w==
+-----END CERTIFICATE-----
+ 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business
+   i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root
+-----BEGIN CERTIFICATE-----
+MIIFPzCCBCegAwIBAgIQDlBz/++iRSmLDeVRHT/hADANBgkqhkiG9w0BAQUFADBv
+MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk
+ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF
+eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDcwOTE4MTkyMlow
+gckxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl
 bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u
 ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv
-cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg
-Q0EgU0dDMB4XDTEwMDIxODAwMDAwMFoXDTEyMDIxOTIzNTk1OVowgcsxCzAJBgNV
-BAYTAkZSMQ4wDAYDVQQREwUxNDAwMDERMA8GA1UECBMIQ2FsdmFkb3MxDTALBgNV
-BAcTBENBRU4xGzAZBgNVBAkTEjIyIHJ1ZSBkZSBCcmV0YWduZTEVMBMGA1UEChMM
-VEJTIElOVEVSTkVUMRcwFQYDVQQLEw4wMDAyIDQ0MDQ0MzgxMDEbMBkGA1UECxMS
-c2hhLTI1NiBwcm9kdWN0aW9uMSAwHgYDVQQDExdzaGEyNTYudGJzLWludGVybmV0
-LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKbuM8VT7f0nntwu
-N3F7v9KIBlhKNAxqCrziOXU5iqUt8HrQB3DtHbdmII+CpVUlwlmepsx6G+srEZ9a
-MIGAy0nxi5aLb7watkyIdPjJTMvTUBQ/+RPWzt5JtYbbY9BlJ+yci0dctP74f4NU
-ISLtlrEjUbf2gTohLrcE01TfmOF6PDEbB5PKDi38cB3NzKfizWfrOaJW6Q1C1qOJ
-y4/4jkUREX1UFUIxzx7v62VfjXSGlcjGpBX1fvtABQOSLeE0a6gciDZs1REqroFf
-5eXtqYphpTa14Z83ITXMfgg5Nze1VtMnzI9Qx4blYBw4dgQVEuIsYr7FDBOITDzc
-VEVXZx0CAwEAAaOCAj8wggI7MB8GA1UdIwQYMBaAFAdEdoWTKLx/bXjSCuv6TEvf
-2YIfMB0GA1UdDgQWBBSJKI/AYVI9RQNY0QPIqc8ej2QivTAOBgNVHQ8BAf8EBAMC
-BaAwDAYDVR0TAQH/BAIwADA0BgNVHSUELTArBggrBgEFBQcDAQYIKwYBBQUHAwIG
-CisGAQQBgjcKAwMGCWCGSAGG+EIEATBMBgNVHSAERTBDMEEGCysGAQQBgOU3AgQB
-MDIwMAYIKwYBBQUHAgEWJGh0dHBzOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0Ev
-Q1BTNDBtBgNVHR8EZjBkMDKgMKAuhixodHRwOi8vY3JsLnRicy1pbnRlcm5ldC5j
-b20vVEJTWDUwOUNBU0dDLmNybDAuoCygKoYoaHR0cDovL2NybC50YnMteDUwOS5j
-b20vVEJTWDUwOUNBU0dDLmNybDCBpgYIKwYBBQUHAQEEgZkwgZYwOAYIKwYBBQUH
-MAKGLGh0dHA6Ly9jcnQudGJzLWludGVybmV0LmNvbS9UQlNYNTA5Q0FTR0MuY3J0
-MDQGCCsGAQUFBzAChihodHRwOi8vY3J0LnRicy14NTA5LmNvbS9UQlNYNTA5Q0FT
-R0MuY3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wPwYD
-VR0RBDgwNoIXc2hhMjU2LnRicy1pbnRlcm5ldC5jb22CG3d3dy5zaGEyNTYudGJz
-LWludGVybmV0LmNvbTANBgkqhkiG9w0BAQsFAAOCAQEAA5NL0D4QSqhErhlkdPmz
-XtiMvdGL+ZehM4coTRIpasM/Agt36Rc0NzCvnQwKE+wkngg1Gy2qe7Q0E/ziqBtB
-fZYzdVgu1zdiL4kTaf+wFKYAFGsFbyeEmXysy+CMwaNoF2vpSjCU1UD56bEnTX/W
-fxVZYxtBQUpnu2wOsm8cDZuZRv9XrYgAhGj9Tt6F0aVHSDGn59uwShG1+BVF/uju
-SCyPTTjL1oc7YElJUzR/x4mQJYvtQI8gDIDAGEOs7v3R/gKa5EMfbUQUI4C84UbI
-Yz09Jdnws/MkC/Hm1BZEqk89u7Hvfv+oHqEb0XaUo0TDfsxE0M1sMdnLb91QNQBm
-UQ==
------END CERTIFICATE-----
- 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC
-   i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root
------BEGIN CERTIFICATE-----
-MIIFVjCCBD6gAwIBAgIQXpDZ0ETJMV02WTx3GTnhhTANBgkqhkiG9w0BAQUFADBv
-MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk
-ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF
-eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDYyNDE5MDYzMFow
-gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl
-bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u
-ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv
-cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg
-Q0EgU0dDMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgOkO3f7wzN6
-rOjg45tR5vjBfzK7qmV9IBxb/QW9EEXxG+E7FNhZqQLtwGBKoSsHTnQqV75wWMk0
-9tinWvftBkSpj5sTi/8cbzJfUvTSVYh3Qxv6AVVjMMH/ruLjE6y+4PoaPs8WoYAQ
-ts5R4Z1g8c/WnTepLst2x0/Wv7GmuoQi+gXvHU6YrBiu7XkeYhzc95QdviWSJRDk
-owhb5K43qhcvjRmBfO/paGlCliDGZp8mHwrI21mwobWpVjTxZRwYO3bd4+TGcI4G
-Ie5wmHwE8F7SK1tgSqbBacKjDa93j7txKkfz/Yd2n7TGqOXiHPsJpG655vrKtnXk
-9vs1zoDeJQIDAQABo4IBljCCAZIwHQYDVR0OBBYEFAdEdoWTKLx/bXjSCuv6TEvf
-2YIfMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMCAGA1UdJQQZ
-MBcGCisGAQQBgjcKAwMGCWCGSAGG+EIEATAYBgNVHSAEETAPMA0GCysGAQQBgOU3
-AgQBMHsGA1UdHwR0MHIwOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0Fk
-ZFRydXN0RXh0ZXJuYWxDQVJvb3QuY3JsMDagNKAyhjBodHRwOi8vY3JsLmNvbW9k
-by5uZXQvQWRkVHJ1c3RFeHRlcm5hbENBUm9vdC5jcmwwgYAGCCsGAQUFBwEBBHQw
-cjA4BggrBgEFBQcwAoYsaHR0cDovL2NydC5jb21vZG9jYS5jb20vQWRkVHJ1c3RV
-VE5TR0NDQS5jcnQwNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQuY29tb2RvLm5ldC9B
-ZGRUcnVzdFVUTlNHQ0NBLmNydDARBglghkgBhvhCAQEEBAMCAgQwDQYJKoZIhvcN
-AQEFBQADggEBAK2zEzs+jcIrVK9oDkdDZNvhuBYTdCfpxfFs+OAujW0bIfJAy232
-euVsnJm6u/+OrqKudD2tad2BbejLLXhMZViaCmK7D9nrXHx4te5EP8rL19SUVqLY
-1pTnv5dhNgEgvA7n5lIzDSYs7yRLsr7HJsYPr6SeYSuZizyX1SNz7ooJ32/F3X98
-RB0Mlc/E0OyOrkQ9/y5IrnpnaSora8CnUrV5XNOg+kyCz9edCyx4D5wXYcwZPVWz
-8aDqquESrezPyjtfi4WRO4s/VD3HLZvOxzMrWAVYCDG9FxaOhF0QGuuG1F7F3GKV
-v6prNyCl016kRl2j1UT+a7gLd8fA25A4C9E=
+cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEdMBsGA1UEAxMUVEJTIFg1MDkg
+Q0EgYnVzaW5lc3MwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDB1PAU
+qudCcz3tmyGcf+u6EkZqonKKHrV4gZYbvVkIRojmmlhfi/jwvpHvo8bqSt/9Rj5S
+jhCDW0pcbI+IPPtD1Jy+CHNSfnMqVDy6CKQ3p5maTzCMG6ZT+XjnvcND5v+FtaiB
+xk1iCX6uvt0jeUtdZvYbyytsSDE6c3Y5//wRxOF8tM1JxibwO3pyER26jbbN2gQz
+m/EkdGjLdJ4svPk23WDAvQ6G0/z2LcAaJB+XLfqRwfQpHQvfKa1uTi8PivC8qtip
+rmNQMMPMjxSK2azX8cKjjTDJiUKaCb4VHlJDWKEsCFRpgJAoAuX8f7Yfs1M4esGo
+sWb3PGspK3O22uIlAgMBAAGjggF6MIIBdjAdBgNVHQ4EFgQUGgkEzPkJj7seoM7U
+pBR/S9UjWM0wDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwGAYD
+VR0gBBEwDzANBgsrBgEEAYDlNwIBATB7BgNVHR8EdDByMDigNqA0hjJodHRwOi8v
+Y3JsLmNvbW9kb2NhLmNvbS9BZGRUcnVzdEV4dGVybmFsQ0FSb290LmNybDA2oDSg
+MoYwaHR0cDovL2NybC5jb21vZG8ubmV0L0FkZFRydXN0RXh0ZXJuYWxDQVJvb3Qu
+Y3JsMIGGBggrBgEFBQcBAQR6MHgwOwYIKwYBBQUHMAKGL2h0dHA6Ly9jcnQuY29t
+b2RvY2EuY29tL0FkZFRydXN0VVROU2VydmVyQ0EuY3J0MDkGCCsGAQUFBzAChi1o
+dHRwOi8vY3J0LmNvbW9kby5uZXQvQWRkVHJ1c3RVVE5TZXJ2ZXJDQS5jcnQwEQYJ
+YIZIAYb4QgEBBAQDAgIEMA0GCSqGSIb3DQEBBQUAA4IBAQA7mqrMgk/MrE6QnbNA
+h4nRCn2ti4bg4w2C3lB6bSvRPnYwuNw9Jb8vuKkNFzRDxNJXqVDZdfFW5CVQJuyd
+nfAx83+wk+spzvFaE1KhFYfN9G9pQfXUfvDRoIcJgPEKUXL1wRiOG+IjU3VVI8pg
+IgqHkr7ylln5i5zCiFAPuIJmYUSFg/gxH5xkCNcjJqqrHrHatJr6Qrrke93joupw
+oU1njfAcZtYp6fbiK6u2b1pJqwkVBE8RsfLnPhRj+SFbpvjv8Od7o/ieJhFIYQNU
+k2jX2u8qZnAiNw93LZW9lpYjtuvMXq8QQppENNja5b53q7UwI+lU7ZGjZ7quuESp
+J6/5
 -----END CERTIFICATE-----
  2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root
-   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC
+   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware
 -----BEGIN CERTIFICATE-----
-MIIEZjCCA06gAwIBAgIQUSYKkxzif5zDpV954HKugjANBgkqhkiG9w0BAQUFADCB
-kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
+MIIETzCCAzegAwIBAgIQHM5EYpUZep1jUvnyI6m2mDANBgkqhkiG9w0BAQUFADCB
+lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
 Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho
-dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw
-IFNHQzAeFw0wNTA2MDcwODA5MTBaFw0xOTA2MjQxOTA2MzBaMG8xCzAJBgNVBAYT
-AlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0
-ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB
-IFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC39xoz5vIABC05
-4E5b7R+8bA/Ntfojts7emxEzl6QpTH2Tn71KvJPtAxrjj8/lbVBa1pcplFqAsEl6
-2y6V/bjKvzc4LR4+kUGtcFbH8E8/6DKedMrIkFTpxl8PeJ2aQDwOrGGqXhSPnoeh
-alDc15pOrwWzpnGUnHGzUGAKxxOdOAeGAqjpqGkmGJCrTLBPI6s6T4TY386f4Wlv
-u9dC12tE5Met7m1BX3JacQg3s3llpFmglDf3AC8NwpJy2tA4ctsUqEXEXSp9t7TW
-xO6szRNEt8kr3UMAJfphuWlqWCMRt6czj1Z1WfXNKddGtworZbbTQm8Vsrh7++/p
-XVPVNFonAgMBAAGjgdgwgdUwHwYDVR0jBBgwFoAUUzLRs89/+uDxoF2FTpLSnkUd
-tE8wHQYDVR0OBBYEFK29mHo0tCb3+sQmVO8DveAky1QaMA4GA1UdDwEB/wQEAwIB
-BjAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBAjAgBgNVHSUEGTAX
-BgorBgEEAYI3CgMDBglghkgBhvhCBAEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDov
-L2NybC51c2VydHJ1c3QuY29tL1VUTi1EQVRBQ29ycFNHQy5jcmwwDQYJKoZIhvcN
-AQEFBQADggEBAMbuUxdoFLJRIh6QWA2U/b3xcOWGLcM2MY9USEbnLQg3vGwKYOEO
-rVE04BKT6b64q7gmtOmWPSiPrmQH/uAB7MXjkesYoPF1ftsK5p+R26+udd8jkWjd
-FwBaS/9kbHDrARrQkNnHptZt9hPk/7XJ0h4qy7ElQyZ42TCbTg0evmnv3+r+LbPM
-+bDdtRTKkdSytaX7ARmjR3mfnYyVhzT4HziS2jamEfpr62vp3EV4FTkG101B5CHI
-3C+H0be/SGB1pWLLJN47YaApIKa+xWycxOkKaSLvkTr6Jq/RW0GnOuL4OAdCq8Fb
-+M5tug8EPzI0rNwEKNdwMBQmBsTkm5jVz3g=
+dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt
+SGFyZHdhcmUwHhcNMDUwNjA3MDgwOTEwWhcNMTkwNzA5MTgxOTIyWjBvMQswCQYD
+VQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0
+IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5h
+bCBDQSBSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/caM+by
+AAQtOeBOW+0fvGwPzbX6I7bO3psRM5ekKUx9k5+9SryT7QMa44/P5W1QWtaXKZRa
+gLBJetsulf24yr83OC0ePpFBrXBWx/BPP+gynnTKyJBU6cZfD3idmkA8Dqxhql4U
+j56HoWpQ3NeaTq8Fs6ZxlJxxs1BgCscTnTgHhgKo6ahpJhiQq0ywTyOrOk+E2N/O
+n+Fpb7vXQtdrROTHre5tQV9yWnEIN7N5ZaRZoJQ39wAvDcKSctrQOHLbFKhFxF0q
+fbe01sTurM0TRLfJK91DACX6YblpalgjEbenM49WdVn1zSnXRrcKK2W200JvFbK4
+e/vv6V1T1TRaJwIDAQABo4G9MIG6MB8GA1UdIwQYMBaAFKFyXyYbKJhDlV0HN9WF
+lp1L0sNFMB0GA1UdDgQWBBStvZh6NLQm9/rEJlTvA73gJMtUGjAOBgNVHQ8BAf8E
+BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhCAQEEBAMCAQIwRAYDVR0f
+BD0wOzA5oDegNYYzaHR0cDovL2NybC51c2VydHJ1c3QuY29tL1VUTi1VU0VSRmly
+c3QtSGFyZHdhcmUuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQByQhANOs4kClrwF8BW
+onvUOGCSjRK52zYZgDXYNjDtmr5rJ6NyPFDNn+JxkLpjYetIFMTbSRe679Bt8m7a
+gIAoQYFQtxMuyLnJegB2aEbQiIxh/tC21UcFF7ktdnDoTlA6w3pLuvunaI84Of3o
+2YBrhzkTbCfaYk5JRlTpudW9DkUkHBsyx3nknPKnplkIGaK0jgn8E0n+SFabYaHk
+I9LroYT/+JtLefh9lgBdAgVv0UPbzoGfuDsrk/Zh+UrgbLFpHoVnElhzbkh64Z0X
+OGaJunQc68cCZu5HTn/aK7fBGMcVflRCXLVEQpU9PIAdGA8Ynvg684t8GMaKsRl1
+jIGZ
 -----END CERTIFICATE-----
- 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC
-   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC
+ 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware
+   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware
 -----BEGIN CERTIFICATE-----
-MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB
-kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
+MIIEdDCCA1ygAwIBAgIQRL4Mi1AAJLQR0zYq/mUK/TANBgkqhkiG9w0BAQUFADCB
+lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
 Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho
-dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw
-IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG
-EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD
-VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu
-dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN
-BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6
-E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ
-D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK
-4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq
-lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW
-bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB
-o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT
-MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js
-LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr
-BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB
-AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft
-Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj
-j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH
-KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv
-2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3
-mfnGV/TJVTl4uix5yaaIK/QI
+dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt
+SGFyZHdhcmUwHhcNOTkwNzA5MTgxMDQyWhcNMTkwNzA5MTgxOTIyWjCBlzELMAkG
+A1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0eTEe
+MBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8v
+d3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdh
+cmUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCx98M4P7Sof885glFn
+0G2f0v9Y8+efK+wNiVSZuTiZFvfgIXlIwrthdBKWHTxqctU8EGc6Oe0rE81m65UJ
+M6Rsl7HoxuzBdXmcRl6Nq9Bq/bkqVRcQVLMZ8Jr28bFdtqdt++BxF2uiiPsA3/4a
+MXcMmgF6sTLjKwEHOG7DpV4jvEWbe1DByTCP2+UretNb+zNAHqDVmBe8i4fDidNd
+oI6yqqr2jmmIBsX6iSHzCJ1pLgkzmykNRg+MzEk0sGlRvfkGzWitZky8PqxhvQqI
+DsjfPe58BEydCl5rkdbux+0ojatNh4lz0G6k0B4WixThdkQDf2Os5M1JnMWS9Ksy
+oUhbAgMBAAGjgbkwgbYwCwYDVR0PBAQDAgHGMA8GA1UdEwEB/wQFMAMBAf8wHQYD
+VR0OBBYEFKFyXyYbKJhDlV0HN9WFlp1L0sNFMEQGA1UdHwQ9MDswOaA3oDWGM2h0
+dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VVE4tVVNFUkZpcnN0LUhhcmR3YXJlLmNy
+bDAxBgNVHSUEKjAoBggrBgEFBQcDAQYIKwYBBQUHAwUGCCsGAQUFBwMGBggrBgEF
+BQcDBzANBgkqhkiG9w0BAQUFAAOCAQEARxkP3nTGmZev/K0oXnWO6y1n7k57K9cM
+//bey1WiCuFMVGWTYGufEpytXoMs61quwOQt9ABjHbjAbPLPSbtNk28Gpgoiskli
+CE7/yMgUsogWXecB5BKV5UU0s4tpvc+0hY91UZ59Ojg6FEgSxvunOxqNDYJAB+gE
+CJChicsZUN/KHAG8HQQZexB2lzvukJDKxA4fFm517zP4029bHpbj4HR3dHuKom4t
+3XbWOTCC8KucUvIqx69JXn7HaOWCgchqJ/kniCrVWFCVH/A7HFe7fRQ5YiuayZSS
+KqMiDP+JJn1fIytH1xUdqWqeUQ0qUZ6B+dQ7XnASfxAynB67nfhmqA==
 -----END CERTIFICATE-----
diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py
--- a/lib-python/2.7/test/test_os.py
+++ b/lib-python/2.7/test/test_os.py
@@ -275,7 +275,7 @@
         try:
             result.f_bfree = 1
             self.fail("No exception thrown")
-        except TypeError:
+        except (TypeError, AttributeError):
             pass
 
         try:
diff --git a/lib-python/2.7/test/test_socket.py b/lib-python/2.7/test/test_socket.py
--- a/lib-python/2.7/test/test_socket.py
+++ b/lib-python/2.7/test/test_socket.py
@@ -1066,6 +1066,9 @@
         def recv(self, size):
             return self._recv_step.next()()
 
+        def _reuse(self): pass
+        def _drop(self): pass
+
     @staticmethod
     def _raise_eintr():
         raise socket.error(errno.EINTR)
@@ -1321,7 +1324,8 @@
             closed = False
             def flush(self): pass
             def close(self): self.closed = True
-            def _decref_socketios(self): pass
+            def _reuse(self): pass
+            def _drop(self): pass
 
         # must not close unless we request it: the original use of _fileobject
         # by module socket requires that the underlying socket not be closed until
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -111,13 +111,12 @@
         if test_support.verbose:
             sys.stdout.write("\n" + pprint.pformat(p) + "\n")
         self.assertEqual(p['subject'],
-                         ((('countryName', u'US'),),
-                          (('stateOrProvinceName', u'Delaware'),),
-                          (('localityName', u'Wilmington'),),
-                          (('organizationName', u'Python Software Foundation'),),
-                          (('organizationalUnitName', u'SSL'),),
-                          (('commonName', u'somemachine.python.org'),)),
+                         ((('countryName', 'XY'),),
+                          (('localityName', 'Castle Anthrax'),),
+                          (('organizationName', 'Python Software Foundation'),),
+                          (('commonName', 'localhost'),))
                         )
+        self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),))
         # Issue #13034: the subjectAltName in some certificates
         # (notably projects.developer.nokia.com:443) wasn't parsed
         p = ssl._ssl._test_decode_cert(NOKIACERT)
diff --git a/lib-python/2.7/test/test_urllib2.py b/lib-python/2.7/test/test_urllib2.py
--- a/lib-python/2.7/test/test_urllib2.py
+++ b/lib-python/2.7/test/test_urllib2.py
@@ -270,6 +270,8 @@
         self.reason = reason
     def read(self):
         return ''
+    def _reuse(self): pass
+    def _drop(self): pass
 
 class MockHTTPClass:
     def __init__(self):
diff --git a/lib-python/2.7/urllib2.py b/lib-python/2.7/urllib2.py
--- a/lib-python/2.7/urllib2.py
+++ b/lib-python/2.7/urllib2.py
@@ -1193,6 +1193,8 @@
         # out of socket._fileobject() and into a base class.
 
         r.recv = r.read
+        r._reuse = lambda: None
+        r._drop = lambda: None
         fp = socket._fileobject(r, close=True)
 
         resp = addinfourl(fp, r.msg, req.get_full_url())
diff --git a/lib-python/2.7/uuid.py b/lib-python/2.7/uuid.py
--- a/lib-python/2.7/uuid.py
+++ b/lib-python/2.7/uuid.py
@@ -44,6 +44,8 @@
     UUID('00010203-0405-0607-0809-0a0b0c0d0e0f')
 """
 
+import struct
+
 __author__ = 'Ka-Ping Yee <ping at zesty.ca>'
 
 RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, RESERVED_FUTURE = [
@@ -125,25 +127,39 @@
         overriding the given 'hex', 'bytes', 'bytes_le', 'fields', or 'int'.
         """
 
-        if [hex, bytes, bytes_le, fields, int].count(None) != 4:
-            raise TypeError('need one of hex, bytes, bytes_le, fields, or int')
         if hex is not None:
+            if (bytes is not None or bytes_le is not None or
+                    fields is not None or int is not None):
+                raise TypeError('if the hex argument is given, bytes,'
+                                ' bytes_le, fields,  and int need to be None')
             hex = hex.replace('urn:', '').replace('uuid:', '')
             hex = hex.strip('{}').replace('-', '')
             if len(hex) != 32:
                 raise ValueError('badly formed hexadecimal UUID string')
             int = long(hex, 16)
-        if bytes_le is not None:
+        elif bytes_le is not None:
+            if bytes is not None or fields is not None or int is not None:
+                raise TypeError('if the bytes_le argument is given, bytes,'
+                                ' fields, and int need to be None')
             if len(bytes_le) != 16:
                 raise ValueError('bytes_le is not a 16-char string')
             bytes = (bytes_le[3] + bytes_le[2] + bytes_le[1] + bytes_le[0] +
                      bytes_le[5] + bytes_le[4] + bytes_le[7] + bytes_le[6] +
                      bytes_le[8:])
-        if bytes is not None:
+            int = (struct.unpack('>Q', bytes[:8])[0] << 64 |
+                   struct.unpack('>Q', bytes[8:])[0])
+        elif bytes is not None:
+            if fields is not None or int is not None:
+                raise TypeError('if the bytes argument is given, fields '
+                                'and int need to be None')
             if len(bytes) != 16:
                 raise ValueError('bytes is not a 16-char string')
-            int = long(('%02x'*16) % tuple(map(ord, bytes)), 16)
-        if fields is not None:
+            int = (struct.unpack('>Q', bytes[:8])[0] << 64 |
+                   struct.unpack('>Q', bytes[8:])[0])
+        elif fields is not None:
+            if int is not None:
+                raise TypeError('if the fields argument is given, int needs'
+                                ' to be None')
             if len(fields) != 6:
                 raise ValueError('fields is not a 6-tuple')
             (time_low, time_mid, time_hi_version,
@@ -163,9 +179,12 @@
             clock_seq = (clock_seq_hi_variant << 8L) | clock_seq_low
             int = ((time_low << 96L) | (time_mid << 80L) |
                    (time_hi_version << 64L) | (clock_seq << 48L) | node)
-        if int is not None:
+        elif int is not None:
             if not 0 <= int < 1<<128L:
                 raise ValueError('int is out of range (need a 128-bit value)')
+        else:
+            raise TypeError('one of hex, bytes, bytes_le, fields,'
+                            ' or int need to be not None')
         if version is not None:
             if not 1 <= version <= 5:
                 raise ValueError('illegal version number')
@@ -175,7 +194,7 @@
             # Set the version number.
             int &= ~(0xf000 << 64L)
             int |= version << 76L
-        self.__dict__['int'] = int
+        object.__setattr__(self, 'int', int)
 
     def __cmp__(self, other):
         if isinstance(other, UUID):
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -1,6 +1,9 @@
 """Reimplementation of the standard extension module '_curses' using cffi."""
 
 import sys
+if sys.platform == 'win32':
+    #This module does not exist in windows
+    raise ImportError('No module named _curses')
 from functools import wraps
 
 from cffi import FFI
@@ -966,7 +969,7 @@
     r, g, b = ffi.new("short *"), ffi.new("short *"), ffi.new("short *")
     if lib.color_content(color, r, g, b) == lib.ERR:
         raise error("Argument 1 was out of range. Check value of COLORS.")
-    return (r, g, b)
+    return (r[0], g[0], b[0])
 
 
 def color_pair(n):
@@ -1121,6 +1124,7 @@
         term = ffi.NULL
     err = ffi.new("int *")
     if lib.setupterm(term, fd, err) == lib.ERR:
+        err = err[0]
         if err == 0:
             raise error("setupterm: could not find terminal")
         elif err == -1:
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -165,6 +165,63 @@
 Nyy rkprcgoybpxf frrz fnar.
 N cvax tyvggrel ebgngvat ynzoqn
 "vg'f yvxryl grzcbenel hagvy sberire" nevtb
+"Lbh xabj jung'f avpr nobhg EClguba?  Ybatre fjbeq svtugf."
+nccneragyl pbashfvba vf n srngher
+nccneragyl pbashfvba vf n srngher... be vf vg?
+ClCl 1.7 eryrnfrq
+vs lbh jnag gb or penml, lbh qba'g unir gb sbepr vg
+vs lbh jnag vg gb or iveghny, lbh fubhyq abg sbepr vg
+<Nyrk_Tnlabe> svwny: V whfg... fgnegrq pbqvat naq fhqqragyl... nobzvangvba
+fabj, fabj! :-)
+fabj, fabj, fabj, fabj
+clcl 1.8 eryrnfrq
+vg jnf srj lnxf gbb yngr
+ClCl vf n enpr orgjrra hf funivat lnxf, naq gur havirefr gelvat gb cebqhpr zber naq zber lnxf
+Jevgvat na SC7 nabalzbhf cebcbfny sbe ClCl vf yvxr znxvat n gi nq sbe n uvtu cresbeznapr fcbegf pne jvgubhg orvat noyr gb zragvba vgf zbqry be znahsnpghere
+Fabj, fabj (ntnva)
+Fgvyy fabjvat
+thrff jung, fabj
+"lbh haqrerfgvzngr gur vzcbegnapr bs zvpebnepuvgrpgher" "ab, vg'f zber gung jr ner fgvyy unccvyl va gur ynaq bs ernpunoyr sehvg"
+Jub nz V?  Naq vs lrf, ubj znal?
+ClCl vf nyjnlf n cynfzn
+"genafyngvba gbbypunva" = "EClgure"?  gb jevgr vagrEClguref va
+"sberire" va clcl grezf zrnaf yvxr srj zbaguf :)
+"Onu.  PClguba bofphevgl. - Nezva Evtb"
+svwny: pna V vavgvngr lbh ba gur (rnfl ohg) aba-gevivny gbcvp bs jevgvat P shapgvba glcrf? :-)
+nyy fbsgjner vzcebirzragf unccra ol n ovg
+gur genprf qba'g yvr
+:-) be engure ":-/" bss-ol-bar-xrl reebe
+Be Nezva Evtb. V guvax ur'f noyr gb haqrefgnaq k86 gur jnl Plcure pna frr jbzra va gur zngevk.
+V zvtug, ohg abobql erfcrpgf zr
+cerohvyg vafgnapr Ryyvcfvf unf ab nggevohgr 'reeab'
+guvf frnfba'f svefg "fabj! fabj!" vf urer
+ClCl 2.0 orgn1 eryrnfrq - orggre yngr guna arire
+Fjvgreynaq 2012: zber fabj va Qrprzore guna rire fvapr clcl fgnegrq
+Fjvgmreynaq 2012: zber fabj va Qrprzore guna rire fvapr clcl fgnegrq
+<nevtngb> n sngny reebe, ol qrsvavgvba, vf sngny
+<nagbphav> V'z tynq gung jr cebtenz va Clguba naq jr qba'g qrny jvgu gubfr vffhrf rirel qnl. Ncneg gur snpg gung jr unir gb qrny jvgu gurz naljnl, vg frrzf
+unccl arj lrne!
+"zrffl" vf abg n whqtrzrag, ohg whfg n snpg bs pbzcyvpngrqarff
+n ybg bs fabj
+qb lbh xabj nobhg n gbnfgre jvgu 8XO bs ENZ naq 64XO bs EBZ?
+vg'f orra fabjvat rirelqnl sbe n juvyr, V pna'g whfg chg "fabj, fabj" hc urer rirel qnl
+fabjonyy svtugf!
+sbejneq pbzcngvovyvgl jvgu bcgvzvmngvbaf gung unira'g orra vairagrq lrg
+jr fgvyy unir gb jevgr fbsgjner jvgu n zrgnfcnpr ohooyr va vg
+cebonoyl gur ynfg gvzr va gur frnfba, ohg: fabj, fabj!
+ClCl 2.0-orgn2 eryrnfrq
+Gur ceboyrz vf gung sbe nyzbfg nal aba-gevivny cebtenz, vg'f abg pyrne jung 'pbeerpg' zrnaf.
+ClCl 2.0 nyzbfg eryrnfrq
+ClCl 2.0 eryrnfrq
+WVG pbzcvyref fubhyq or jevggra ol crbcyr jub npghnyyl unir snvgu va WVG pbzcvyref' novyvgl gb znxrf guvatf tb fpernzvat snfg
+ClCl 2.0.1 eryrnfrq
+arire haqrerfgvzngr gur vzcebonoyr jura lbh qb fbzrguvat ng 2TUm
+ClCl 2.0.2 eryrnfrq
+nyy jr arrq vf n angvir Cebybt znpuvar
+V haqrefgnaq ubj qravnyvfz vf n onq qrohttvat grpuavdhr
+rirel IZ fubhyq pbzr jvgu arheny argjbex genvarq gb erpbtavmr zvpeborapuznexf naq enaqbzyl syhpghngr gurz +/-9000%
+lbh qvq abg nccebnpu clcl sebz gur rnfl raq: fgz, gura wvg. vg'f n ovg gur Abegu snpr
+va ClCl orvat bayl zbqrengryl zntvp vf n tbbq guvat <psobym>
 """
 
 from string import ascii_uppercase, ascii_lowercase
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -363,9 +363,11 @@
     pass
 
 
-def connect(database, **kwargs):
-    factory = kwargs.get("factory", Connection)
-    return factory(database, **kwargs)
+def connect(database, timeout=5.0, detect_types=0, isolation_level="",
+                 check_same_thread=True, factory=None, cached_statements=100):
+    factory = Connection if not factory else factory
+    return factory(database, timeout, detect_types, isolation_level,
+                    check_same_thread, factory, cached_statements)
 
 
 def _unicode_text_factory(x):
@@ -1229,7 +1231,10 @@
         if cvt is not None:
             param = cvt(param)
 
-        param = adapt(param)
+        try:
+            param = adapt(param)
+        except:
+            pass  # And use previous value
 
         if param is None:
             rc = _lib.sqlite3_bind_null(self._statement, idx)
@@ -1305,7 +1310,7 @@
         for i in xrange(_lib.sqlite3_column_count(self._statement)):
             name = _lib.sqlite3_column_name(self._statement, i)
             if name:
-                name = _ffi.string(name).decode('utf-8').split("[")[0].strip()
+                name = _ffi.string(name).split("[")[0].strip()
             desc.append((name, None, None, None, None, None, None))
         return desc
 
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
--- a/lib_pypy/_tkinter/__init__.py
+++ b/lib_pypy/_tkinter/__init__.py
@@ -22,6 +22,7 @@
 READABLE = tklib.TCL_READABLE
 WRITABLE = tklib.TCL_WRITABLE
 EXCEPTION = tklib.TCL_EXCEPTION
+DONT_WAIT = tklib.TCL_DONT_WAIT
 
 def create(screenName=None, baseName=None, className=None,
            interactive=False, wantobjects=False, wantTk=True,
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -4,7 +4,23 @@
 from . import TclError
 from .tclobj import TclObject, FromObj, AsObj, TypeCache
 
+import contextlib
 import sys
+import threading
+import time
+
+
+class _DummyLock(object):
+    "A lock-like object that does not do anything"
+    def acquire(self):
+        pass
+    def release(self):
+        pass
+    def __enter__(self):
+        pass
+    def __exit__(self, *exc):
+        pass
+
 
 def varname_converter(input):
     if isinstance(input, TclObject):
@@ -37,17 +53,18 @@
     def PythonCmd(clientData, interp, argc, argv):
         self = tkffi.from_handle(clientData)
         assert self.app.interp == interp
-        try:
-            args = [tkffi.string(arg) for arg in argv[1:argc]]
-            result = self.func(*args)
-            obj = AsObj(result)
-            tklib.Tcl_SetObjResult(interp, obj)
-        except:
-            self.app.errorInCmd = True
-            self.app.exc_info = sys.exc_info()
-            return tklib.TCL_ERROR
-        else:
-            return tklib.TCL_OK
+        with self.app._tcl_lock_released():
+            try:
+                args = [tkffi.string(arg) for arg in argv[1:argc]]
+                result = self.func(*args)
+                obj = AsObj(result)
+                tklib.Tcl_SetObjResult(interp, obj)
+            except:
+                self.app.errorInCmd = True
+                self.app.exc_info = sys.exc_info()
+                return tklib.TCL_ERROR
+            else:
+                return tklib.TCL_OK
 
     @tkffi.callback("Tcl_CmdDeleteProc")
     def PythonCmdDelete(clientData):
@@ -58,6 +75,8 @@
 
 
 class TkApp(object):
+    _busywaitinterval = 0.02  # 20ms.
+
     def __new__(cls, screenName, baseName, className,
                 interactive, wantobjects, wantTk, sync, use):
         if not wantobjects:
@@ -73,6 +92,12 @@
         self.quitMainLoop = False
         self.errorInCmd = False
 
+        if not self.threaded:
+            # TCL is not thread-safe, calls needs to be serialized.
+            self._tcl_lock = threading.Lock()
+        else:
+            self._tcl_lock = _DummyLock()
+
         self._typeCache = TypeCache()
         self._commands = {}
 
@@ -133,6 +158,13 @@
         if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
             raise RuntimeError("Calling Tcl from different appartment")
 
+    @contextlib.contextmanager
+    def _tcl_lock_released(self):
+        "Context manager to temporarily release the tcl lock."
+        self._tcl_lock.release()
+        yield
+        self._tcl_lock.acquire()
+
     def loadtk(self):
         # We want to guard against calling Tk_Init() multiple times
         err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
@@ -159,22 +191,25 @@
         flags=tklib.TCL_LEAVE_ERR_MSG
         if global_only:
             flags |= tklib.TCL_GLOBAL_ONLY
-        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
-        if not res:
-            self.raiseTclError()
-        assert self._wantobjects
-        return FromObj(self, res)
+        with self._tcl_lock:
+            res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+            if not res:
+                self.raiseTclError()
+            assert self._wantobjects
+            return FromObj(self, res)
 
     def _setvar(self, name1, value, global_only=False):
         name1 = varname_converter(name1)
+        # XXX Acquire tcl lock???
         newval = AsObj(value)
         flags=tklib.TCL_LEAVE_ERR_MSG
         if global_only:
             flags |= tklib.TCL_GLOBAL_ONLY
-        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
-                                  newval, flags)
-        if not res:
-            self.raiseTclError()
+        with self._tcl_lock:
+            res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                      newval, flags)
+            if not res:
+                self.raiseTclError()
 
     def _unsetvar(self, name1, name2=None, global_only=False):
         name1 = varname_converter(name1)
@@ -183,9 +218,10 @@
         flags=tklib.TCL_LEAVE_ERR_MSG
         if global_only:
             flags |= tklib.TCL_GLOBAL_ONLY
-        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
-        if res == tklib.TCL_ERROR:
-            self.raiseTclError()
+        with self._tcl_lock:
+            res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
 
     def getvar(self, name1, name2=None):
         return self._var_invoke(self._getvar, name1, name2)
@@ -219,9 +255,10 @@
         if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
             raise NotImplementedError("Call from another thread")
 
-        res = tklib.Tcl_CreateCommand(
-            self.interp, cmdName, _CommandData.PythonCmd,
-            clientData, _CommandData.PythonCmdDelete)
+        with self._tcl_lock:
+            res = tklib.Tcl_CreateCommand(
+                self.interp, cmdName, _CommandData.PythonCmd,
+                clientData, _CommandData.PythonCmdDelete)
         if not res:
             raise TclError("can't create Tcl command")
 
@@ -229,7 +266,8 @@
         if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
             raise NotImplementedError("Call from another thread")
 
-        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        with self._tcl_lock:
+            res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
         if res == -1:
             raise TclError("can't delete Tcl command")
 
@@ -256,11 +294,12 @@
                 tklib.Tcl_IncrRefCount(obj)
                 objects[i] = obj
 
-            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
-            if res == tklib.TCL_ERROR:
-                self.raiseTclError()
-            else:
-                result = self._callResult()
+            with self._tcl_lock:
+                res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+                if res == tklib.TCL_ERROR:
+                    self.raiseTclError()
+                else:
+                    result = self._callResult()
         finally:
             for obj in objects:
                 if obj:
@@ -280,17 +319,19 @@
 
     def eval(self, script):
         self._check_tcl_appartment()
-        res = tklib.Tcl_Eval(self.interp, script)
-        if res == tklib.TCL_ERROR:
-            self.raiseTclError()
-        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+        with self._tcl_lock:
+            res = tklib.Tcl_Eval(self.interp, script)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
 
     def evalfile(self, filename):
         self._check_tcl_appartment()
-        res = tklib.Tcl_EvalFile(self.interp, filename)
-        if res == tklib.TCL_ERROR:
-            self.raiseTclError()
-        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+        with self._tcl_lock:
+            res = tklib.Tcl_EvalFile(self.interp, filename)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
 
     def split(self, arg):
         if isinstance(arg, tuple):
@@ -375,7 +416,10 @@
             if self.threaded:
                 result = tklib.Tcl_DoOneEvent(0)
             else:
-                raise NotImplementedError("TCL configured without threads")
+                with self._tcl_lock:
+                    result = tklib.Tcl_DoOneEvent(tklib.TCL_DONT_WAIT)
+                if result == 0:
+                    time.sleep(self._busywaitinterval)
 
             if result < 0:
                 break
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -28,9 +28,11 @@
         return result
 
     elif value.typePtr == typeCache.BooleanType:
-        return result
+        return bool(value.internalRep.longValue)
     elif value.typePtr == typeCache.ByteArrayType:
-        return result
+        size = tkffi.new('int*')
+        data = tklib.Tcl_GetByteArrayFromObj(value, size)
+        return tkffi.buffer(data, size[0])[:]
     elif value.typePtr == typeCache.DoubleType:
         return value.internalRep.doubleValue
     elif value.typePtr == typeCache.IntType:
@@ -50,7 +52,7 @@
             result.append(FromObj(app, tcl_elem[0]))
         return tuple(result)
     elif value.typePtr == typeCache.ProcBodyType:
-        return result
+        pass  # fall through and return tcl object.
     elif value.typePtr == typeCache.StringType:
         buf = tklib.Tcl_GetUnicode(value)
         length = tklib.Tcl_GetCharLength(value)
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -1,6 +1,7 @@
 # C bindings with libtcl and libtk.
 
 from cffi import FFI
+import sys
 
 tkffi = FFI()
 
@@ -18,6 +19,8 @@
 #define TCL_EVAL_DIRECT ...
 #define TCL_EVAL_GLOBAL ...
 
+#define TCL_DONT_WAIT ...
+
 typedef unsigned short Tcl_UniChar;
 typedef ... Tcl_Interp;
 typedef ...* Tcl_ThreadId;
@@ -69,6 +72,7 @@
 int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
 char *Tcl_GetString(Tcl_Obj* objPtr);
 char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+unsigned char *Tcl_GetByteArrayFromObj(Tcl_Obj* objPtr, int* lengthPtr);
 
 Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
 int Tcl_GetCharLength(Tcl_Obj* objPtr);
@@ -102,6 +106,17 @@
 int Tk_GetNumMainWindows();
 """)
 
+# XXX find a better way to detect paths
+# XXX pick up CPPFLAGS and LDFLAGS and add to these paths?
+if sys.platform.startswith("openbsd"):
+    incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include']
+    linklibs = ['tk85', 'tcl85']
+    libdirs = ['/usr/local/lib', '/usr/X11R6/lib']
+else:
+    incdirs=['/usr/include/tcl']
+    linklibs=['tcl', 'tk']
+    libdirs = []
+
 tklib = tkffi.verify("""
 #include <tcl.h>
 #include <tk.h>
@@ -109,6 +124,7 @@
 char *get_tk_version() { return TK_VERSION; }
 char *get_tcl_version() { return TCL_VERSION; }
 """,
-include_dirs=['/usr/include/tcl'],
-libraries=['tcl', 'tk'],
+include_dirs=incdirs,
+libraries=linklibs,
+library_dirs = libdirs
 )
diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info
--- a/lib_pypy/cffi.egg-info
+++ b/lib_pypy/cffi.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: cffi
-Version: 0.6
+Version: 0.7
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.7"
-__version_info__ = (0, 7)
+__version__ = "0.7.2"
+__version_info__ = (0, 7, 2)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -54,7 +54,8 @@
             # _cffi_backend.so compiled.
             import _cffi_backend as backend
             from . import __version__
-            assert backend.__version__ == __version__
+            assert (backend.__version__ == __version__ or
+                    backend.__version__ == __version__[:3])
             # (If you insist you can also try to pass the option
             # 'backend=backend_ctypes.CTypesBackend()', but don't
             # rely on it!  It's probably not going to work well.)
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -30,7 +30,9 @@
         elif result in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
             result = model.PrimitiveType(result)
         else:
-            assert commontype != result
+            if commontype == result:
+                raise api.FFIError("Unsupported type: %r.  Please file a bug "
+                                   "if you think it should be." % (commontype,))
             result = resolve_common_type(result)   # recursively
         assert isinstance(result, model.BaseTypeByIdentity)
         _CACHE[commontype] = result
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -290,13 +290,26 @@
                 # assume a primitive type.  get it from .names, but reduce
                 # synonyms to a single chosen combination
                 names = list(type.names)
-                if names == ['signed'] or names == ['unsigned']:
-                    names.append('int')
-                if names[0] == 'signed' and names != ['signed', 'char']:
-                    names.pop(0)
-                if (len(names) > 1 and names[-1] == 'int'
-                        and names != ['unsigned', 'int']):
-                    names.pop()
+                if names != ['signed', 'char']:    # keep this unmodified
+                    prefixes = {}
+                    while names:
+                        name = names[0]
+                        if name in ('short', 'long', 'signed', 'unsigned'):
+                            prefixes[name] = prefixes.get(name, 0) + 1
+                            del names[0]
+                        else:
+                            break
+                    # ignore the 'signed' prefix below, and reorder the others
+                    newnames = []
+                    for prefix in ('unsigned', 'short', 'long'):
+                        for i in range(prefixes.get(prefix, 0)):
+                            newnames.append(prefix)
+                    if not names:
+                        names = ['int']    # implicitly
+                    if names == ['int']:   # but kill it if 'short' or 'long'
+                        if 'short' in prefixes or 'long' in prefixes:
+                            names = []
+                    names = newnames + names
                 ident = ' '.join(names)
                 if ident == 'void':
                     return model.void_type
@@ -500,8 +513,8 @@
                 self._partial_length = True
                 return None
         #
-        raise api.FFIError("unsupported non-constant or "
-                           "not immediately constant expression")
+        raise api.FFIError("unsupported expression: expected a "
+                           "simple numeric constant")
 
     def _build_enum_type(self, explicit_name, decls):
         if decls is not None:
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -61,7 +61,9 @@
     def load_library(self):
         # import it with the CFFI backend
         backend = self.ffi._backend
-        module = backend.load_library(self.verifier.modulefilename)
+        # needs to make a path that contains '/', on Posix
+        filename = os.path.join(os.curdir, self.verifier.modulefilename)
+        module = backend.load_library(filename)
         #
         # call loading_gen_struct() to get the struct layout inferred by
         # the C compiler
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -40,9 +40,9 @@
 # for all computations.  See the book for algorithms for converting between
 # proleptic Gregorian ordinals and many other calendar systems.
 
-_DAYS_IN_MONTH = [None, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+_DAYS_IN_MONTH = [-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
 
-_DAYS_BEFORE_MONTH = [None]
+_DAYS_BEFORE_MONTH = [-1]
 dbm = 0
 for dim in _DAYS_IN_MONTH[1:]:
     _DAYS_BEFORE_MONTH.append(dbm)
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,3 +1,4 @@
+import sys
 import _continuation
 
 __version__ = "0.4.0"
@@ -5,7 +6,7 @@
 # ____________________________________________________________
 # Exceptions
 
-class GreenletExit(Exception):
+class GreenletExit(BaseException):
     """This special exception does not propagate to the parent greenlet; it
 can be used to kill a single greenlet."""
 
@@ -75,6 +76,15 @@
             # up the 'parent' explicitly.  Good enough, because a Ctrl-C
             # will show that the program is caught in this loop here.)
             target = target.parent
+            # convert a "raise GreenletExit" into "return GreenletExit"
+            if methodname == 'throw':
+                try:
+                    raise baseargs[0], baseargs[1]
+                except GreenletExit, e:
+                    methodname = 'switch'
+                    baseargs = (((e,), {}),)
+                except:
+                    baseargs = sys.exc_info()[:2] + baseargs[2:]
         #
         try:
             unbound_method = getattr(_continulet, methodname)
@@ -147,5 +157,8 @@
     _tls.current = greenlet
     try:
         raise exc, value, tb
+    except GreenletExit, e:
+        res = e
     finally:
         _continuation.permute(greenlet, greenlet.parent)
+    return ((res,), None)
diff --git a/lib_pypy/numpy.py b/lib_pypy/numpy.py
deleted file mode 100644
--- a/lib_pypy/numpy.py
+++ /dev/null
@@ -1,5 +0,0 @@
-raise ImportError(
-    "The 'numpy' module of PyPy is in-development and not complete. "
-    "To try it out anyway, you can either import from 'numpypy', "
-    "or just write 'import numpypy' first in your program and then "
-    "import from 'numpy' as usual.")
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,14 +1,17 @@
-import core
-from core import *
-import lib
-from lib import *
+from . import core
+from .core import *
+from . import lib
+from .lib import *
 
 from __builtin__ import bool, int, long, float, complex, object, unicode, str
-from core import abs, max, min
 
-__all__ = []
+from .core import round, abs, max, min
+
+__version__ = '1.7.0'
+
+__all__ = ['__version__']
 __all__ += core.__all__
 __all__ += lib.__all__
 
-import sys
-sys.modules.setdefault('numpy', sys.modules['numpypy'])
+#import sys
+#sys.modules.setdefault('numpy', sys.modules['numpypy'])
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
--- a/lib_pypy/numpypy/core/__init__.py
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -1,12 +1,17 @@
-import numeric
-from numeric import *
-import fromnumeric
-from fromnumeric import *
-import shape_base
-from shape_base import *
+from __future__ import division, absolute_import, print_function
 
-from fromnumeric import amax as max, amin as min
-from numeric import absolute as abs
+from . import multiarray
+from . import umath
+from . import numeric
+from .numeric import *
+from . import fromnumeric
+from .fromnumeric import *
+from . import shape_base
+from .shape_base import *
+
+from .fromnumeric import amax as max, amin as min, \
+    round_ as round
+from .numeric import absolute as abs
 
 __all__ = []
 __all__ += numeric.__all__
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -1,36 +1,53 @@
-######################################################################    
+######################################################################
 # This is a copy of numpy/core/fromnumeric.py modified for numpypy
 ######################################################################
-# Each name in __all__ was a function in  'numeric' that is now 
-# a method in 'numpy'.
-# When the corresponding method is added to numpypy BaseArray
-# each function should be added as a module function 
-# at the applevel 
-# This can be as simple as doing the following
-#
-# def func(a, ...):
-#     if not hasattr(a, 'func')
-#         a = numpypy.array(a)
-#     return a.func(...)
-#
-######################################################################
-
-import numpypy
-import _numpypy
-
-# Module containing non-deprecated functions borrowed from Numeric.
-__docformat__ = "restructuredtext en"
+"""Module containing non-deprecated functions borrowed from Numeric.
+
+"""
+from __future__ import division, absolute_import, print_function
+
+import types
+
+from . import multiarray as mu
+from . import umath as um
+from . import numerictypes as nt
+from .numeric import asarray, array, asanyarray, concatenate
+from . import _methods
+
+
+# functions that are methods
+__all__ = [
+        'alen', 'all', 'alltrue', 'amax', 'amin', 'any', 'argmax',
+        'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip',
+        'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean',
+        'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put',
+        'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_',
+        'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze',
+        'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var',
+        ]
+
+
+try:
+    _gentype = types.GeneratorType
+except AttributeError:
+    _gentype = type(None)
+
+# save away Python sum
+_sum_ = sum
 
 # functions that are now methods
-__all__ = ['take', 'reshape', 'choose', 'repeat', 'put',
-           'swapaxes', 'transpose', 'sort', 'argsort', 'argmax', 'argmin',
-           'searchsorted', 'alen',
-           'resize', 'diagonal', 'trace', 'ravel', 'nonzero', 'shape',
-           'compress', 'clip', 'sum', 'product', 'prod', 'sometrue', 'alltrue',
-           'any', 'all', 'cumsum', 'cumproduct', 'cumprod', 'ptp', 'ndim',
-           'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze',
-           'amax', 'amin',
-          ]
+def _wrapit(obj, method, *args, **kwds):
+    try:
+        wrap = obj.__array_wrap__
+    except AttributeError:
+        wrap = None
+    result = getattr(asarray(obj), method)(*args, **kwds)
+    if wrap:
+        if not isinstance(result, mu.ndarray):
+            result = asarray(result)
+        result = wrap(result)
+    return result
+
 
 def take(a, indices, axis=None, out=None, mode='raise'):
     """
@@ -46,6 +63,10 @@
         The source array.
     indices : array_like
         The indices of the values to extract.
+
+        .. versionadded:: 1.8.0
+
+        Also allow scalars for indices.
     axis : int, optional
         The axis over which to select values. By default, the flattened
         input array is used.
@@ -85,8 +106,17 @@
     >>> a[indices]
     array([4, 3, 6])
 
+    If `indices` is not one dimensional, the output also has these dimensions.
+
+    >>> np.take(a, [[0, 1], [2, 3]])
+    array([[4, 3],
+           [5, 7]])
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        take = a.take
+    except AttributeError:
+        return _wrapit(a, 'take', indices, axis, out, mode)
+    return take(indices, axis, out, mode)
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -104,16 +134,23 @@
         One shape dimension can be -1. In this case, the value is inferred
         from the length of the array and remaining dimensions.
     order : {'C', 'F', 'A'}, optional
-        Determines whether the array data should be viewed as in C
-        (row-major) order, FORTRAN (column-major) order, or the C/FORTRAN
-        order should be preserved.
+        Read the elements of `a` using this index order, and place the elements
+        into the reshaped array using this index order.  'C' means to
+        read / write the elements using C-like index order, with the last axis index
+        changing fastest, back to the first axis index changing slowest.  'F'
+        means to read / write the elements using Fortran-like index order, with
+        the first index changing fastest, and the last index changing slowest.
+        Note that the 'C' and 'F' options take no account of the memory layout
+        of the underlying array, and only refer to the order of indexing.  'A'
+        means to read / write the elements in Fortran-like index order if `a` is
+        Fortran *contiguous* in memory, C-like order otherwise.
 
     Returns
     -------
     reshaped_array : ndarray
         This will be a new view object if possible; otherwise, it will
-        be a copy.
-
+        be a copy.  Note there is no guarantee of the *memory layout* (C- or
+        Fortran- contiguous) of the returned array.
 
     See Also
     --------
@@ -121,7 +158,6 @@
 
     Notes
     -----
-
     It is not always possible to change the shape of an array without
     copying the data. If you want an error to be raise if the data is copied,
     you should assign the new shape to the shape attribute of the array::
@@ -129,12 +165,39 @@
      >>> a = np.zeros((10, 2))
      # A transpose make the array non-contiguous
      >>> b = a.T
-     # Taking a view makes it possible to modify the shape without modiying the
+     # Taking a view makes it possible to modify the shape without modifying the
      # initial object.
      >>> c = b.view()
      >>> c.shape = (20)
      AttributeError: incompatible shape for a non-contiguous array
 
+    The `order` keyword gives the index ordering both for *fetching* the values
+    from `a`, and then *placing* the values into the output array.  For example,
+    let's say you have an array:
+
+    >>> a = np.arange(6).reshape((3, 2))
+    >>> a
+    array([[0, 1],
+           [2, 3],
+           [4, 5]])
+
+    You can think of reshaping as first raveling the array (using the given
+    index order), then inserting the elements from the raveled array into the
+    new array using the same kind of index ordering as was used for the
+    raveling.
+
+    >>> np.reshape(a, (2, 3)) # C-like index ordering
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(a, (2, 3), order='F') # Fortran-like index ordering
+    array([[0, 4, 3],
+           [2, 1, 5]])
+    >>> np.reshape(np.ravel(a, order='F'), (2, 3), order='F')
+    array([[0, 4, 3],
+           [2, 1, 5]])
 
     Examples
     --------
@@ -148,12 +211,13 @@
     array([[1, 2],
            [3, 4],
            [5, 6]])
-
     """
     assert order == 'C'
-    if not hasattr(a, 'reshape'):
-       a = numpypy.array(a)
-    return a.reshape(newshape)
+    try:
+        reshape = a.reshape
+    except AttributeError:
+        return _wrapit(a, 'reshape', newshape)
+    return reshape(newshape)
 
 
 def choose(a, choices, out=None, mode='raise'):
@@ -275,7 +339,11 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    return _numpypy.choose(a, choices, out, mode)
+    try:
+        choose = a.choose
+    except AttributeError:
+        return _wrapit(a, 'choose', choices, out=out, mode=mode)
+    return choose(choices, out=out, mode=mode)
 
 
 def repeat(a, repeats, axis=None):
@@ -317,7 +385,11 @@
            [3, 4]])
 
     """
-    return _numpypy.repeat(a, repeats, axis)
+    try:
+        repeat = a.repeat
+    except AttributeError:
+        return _wrapit(a, 'repeat', repeats, axis)
+    return repeat(repeats, axis)
 
 
 def put(a, ind, v, mode='raise'):
@@ -368,7 +440,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return a.put(ind, v, mode)
 
 
 def swapaxes(a, axis1, axis2):
@@ -412,7 +484,10 @@
             [3, 7]]])
 
     """
-    swapaxes = a.swapaxes
+    try:
+        swapaxes = a.swapaxes
+    except AttributeError:
+        return _wrapit(a, 'swapaxes', axis1, axis2)
     return swapaxes(axis1, axis2)
 
 
@@ -456,9 +531,158 @@
     """
     if axes is not None:
         raise NotImplementedError('No "axes" arg yet.')
-    if not hasattr(a, 'T'):
-        a = numpypy.array(a)
-    return a.T
+    try:
+        transpose = a.transpose
+    except AttributeError:
+        return _wrapit(a, 'transpose')
+    return transpose()
+
+
+def partition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Return a partitioned copy of an array.
+
+    Creates a copy of the array with its elements rearranged in such a way that
+    the value of the element in kth position is in the position it would be in
+    a sorted array. All elements smaller than the kth element are moved before
+    this element and all equal or greater are moved behind it. The ordering of
+    the elements in the two partitions is undefined.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to be sorted.
+    kth : int or sequence of ints
+        Element index to partition by. The kth value of the element will be in
+        its final sorted position and all smaller elements will be moved before
+        it and all equal or greater elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all elements
+        indexed by kth  of them into their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort. If None, the array is flattened before
+        sorting. The default is -1, which sorts along the last axis.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'.
+    order : list, optional
+        When `a` is a structured array, this argument specifies which fields
+        to compare first, second, and so on.  This list does not need to
+        include all of the fields.
+
+    Returns
+    -------
+    partitioned_array : ndarray
+        Array of the same type and shape as `a`.
+
+    See Also
+    --------
+    ndarray.partition : Method to sort an array in-place.
+    argpartition : Indirect partition.
+    sort : Full sorting
+
+    Notes
+    -----
+    The various selection algorithms are characterized by their average speed,
+    worst case performance, work space size, and whether they are stable. A
+    stable sort keeps items with the same key in the same relative order. The
+    three available algorithms have the following properties:
+
+    ================= ======= ============= ============ =======
+       kind            speed   worst case    work space  stable
+    ================= ======= ============= ============ =======
+    'introselect'        1        O(n)           0         no
+    ================= ======= ============= ============ =======
+
+    All the partition algorithms make temporary copies of the data when
+    partitioning along any but the last axis.  Consequently, partitioning
+    along the last axis is faster and uses less space than partitioning
+    along any other axis.
+
+    The sort order for complex numbers is lexicographic. If both the real
+    and imaginary parts are non-nan then the order is determined by the
+    real parts except when they are equal, in which case the order is
+    determined by the imaginary parts.
+
+    Examples
+    --------
+    >>> a = np.array([3, 4, 2, 1])
+    >>> np.partition(a, 3)
+    array([2, 1, 3, 4])
+
+    >>> np.partition(a, (1, 3))
+    array([1, 2, 3, 4])
+
+    """
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.partition(kth, axis=axis, kind=kind, order=order)
+    return a
+
+
+def argpartition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Perform an indirect partition along the given axis using the algorithm
+    specified by the `kind` keyword. It returns an array of indices of the
+    same shape as `a` that index data along the given axis in partitioned
+    order.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to sort.
+    kth : int or sequence of ints
+        Element index to partition by. The kth element will be in its final
+        sorted position and all smaller elements will be moved before it and
+        all larger elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all of them into
+        their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort.  The default is -1 (the last axis). If None,
+        the flattened array is used.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'
+    order : list, optional
+        When `a` is an array with fields defined, this argument specifies
+        which fields to compare first, second, etc.  Not all fields need be
+        specified.
+
+    Returns
+    -------
+    index_array : ndarray, int
+        Array of indices that partition `a` along the specified axis.
+        In other words, ``a[index_array]`` yields a sorted `a`.
+
+    See Also
+    --------
+    partition : Describes partition algorithms used.
+    ndarray.partition : Inplace partition.
+    argsort : Full indirect sort
+
+    Notes
+    -----
+    See `partition` for notes on the different selection algorithms.
+
+    Examples
+    --------
+    One dimensional array:
+
+    >>> x = np.array([3, 4, 2, 1])
+    >>> x[np.argpartition(x, 3)]
+    array([2, 1, 3, 4])
+    >>> x[np.argpartition(x, (1, 3))]
+    array([1, 2, 3, 4])
+
+    """
+    return a.argpartition(kth, axis, kind=kind, order=order)
+
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -489,6 +713,7 @@
     argsort : Indirect sort.
     lexsort : Indirect stable sort on multiple keys.
     searchsorted : Find elements in a sorted array.
+    partition : Partial sort.
 
     Notes
     -----
@@ -559,7 +784,13 @@
           dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.sort(axis, kind, order)
+    return a
 
 
 def argsort(a, axis=-1, kind='quicksort', order=None):
@@ -595,6 +826,7 @@
     sort : Describes sorting algorithms used.
     lexsort : Indirect stable sort with multiple keys.
     ndarray.sort : Inplace sort.
+    argpartition : Indirect partial sort.
 
     Notes
     -----
@@ -640,7 +872,11 @@
     array([0, 1])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        argsort = a.argsort
+    except AttributeError:
+        return _wrapit(a, 'argsort', axis, kind, order)
+    return argsort(axis, kind, order)
 
 
 def argmax(a, axis=None):
@@ -694,9 +930,11 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmax'):
-        a = numpypy.array(a)
-    return a.argmax()
+    try:
+        argmax = a.argmax
+    except AttributeError:
+        return _wrapit(a, 'argmax')
+    return argmax()
 
 
 def argmin(a, axis=None):
@@ -710,29 +948,37 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmin'):
-        a = numpypy.array(a)
-    return a.argmin()
-
-
-def searchsorted(a, v, side='left'):
+    try:
+        argmin = a.argmin
+    except AttributeError:
+        return _wrapit(a, 'argmin')
+    return argmin()
+

From noreply at buildbot.pypy.org  Wed Oct 16 06:18:21 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 06:18:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: skipped broken test for subarrays
Message-ID: <20131016041821.CCE711C05DF@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67405:97836986a03b
Date: 2013-10-15 23:33 -0400
http://bitbucket.org/pypy/pypy/changeset/97836986a03b/

Log:	skipped broken test for subarrays

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2932,6 +2932,27 @@
         assert (a[0, 0, 0] == 500).all()
         assert a[0, 0, 0].shape == (10,)
 
+    def test_subarray_multiple_rows(self):
+        import numpypy as np
+        descr = [
+            ('x', 'i4', (2,)),
+            ('y', 'f8', (2, 2)),
+            ('z', 'u1')]
+        buf = [
+            # x     y                  z
+            ([3,2], [[6.,4.],[6.,4.]], 8),
+            ([4,3], [[7.,5.],[7.,5.]], 9),
+            ]
+        h = np.array(buf, dtype=descr)
+        assert len(h) == 2
+        skip('broken')  # XXX
+        assert np.array_equal(h['x'], np.array([buf[0][0],
+                                                buf[1][0]], dtype='i4'))
+        assert np.array_equal(h['y'], np.array([buf[0][1],
+                                                buf[1][1]], dtype='f8'))
+        assert np.array_equal(h['z'], np.array([buf[0][2],
+                                                buf[1][2]], dtype='u1'))
+
     def test_multidim_subarray(self):
         from numpypy import dtype, array
 

From noreply at buildbot.pypy.org  Wed Oct 16 06:18:23 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 06:18:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: provide long double based on double
 until rpython support is done
Message-ID: <20131016041823.157121C06E1@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67406:82f81f0ed787
Date: 2013-10-16 00:12 -0400
http://bitbucket.org/pypy/pypy/changeset/82f81f0ed787/

Log:	provide long double based on double until rpython support is done

diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ b/lib_pypy/numpypy/core/arrayprint.py
@@ -247,10 +247,11 @@
     formatdict = {'bool' : _boolFormatter,
                   'int' : IntegerFormat(data),
                   'float' : FloatFormat(data, precision, suppress_small),
-                  'longfloat' : LongFloatFormat(precision),
+                  'longfloat' : FloatFormat(data, precision, suppress_small),
                   'complexfloat' : ComplexFormat(data, precision,
                                                  suppress_small),
-                  'longcomplexfloat' : LongComplexFormat(precision),
+                  'longcomplexfloat' : ComplexFormat(data, precision,
+                                                     suppress_small),
                   'datetime' : DatetimeFormat(data),
                   'timedelta' : TimedeltaFormat(data),
                   'numpystr' : repr_format,
diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -395,6 +395,9 @@
                   ('int_', 'long'),
                   ('uint', 'ulong'),
                   ('cfloat', 'cdouble'),
+                  ('longfloat', 'longdouble'),
+                  ('clongfloat', 'clongdouble'),
+                  ('longcomplex', 'clongdouble'),
                   ('bool_', 'bool'),
                   ('unicode_', 'unicode'),
                   ]
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -1,5 +1,4 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module.micronumpy.interp_boxes import long_double_size, ENABLED_LONG_DOUBLE
 
 
 class MultiArrayModule(MixedModule):
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -20,14 +20,14 @@
 MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
 MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else ()
 
-# Is this the proper place for this?
-ENABLED_LONG_DOUBLE = False
-long_double_size = rffi.sizeof_c_type('long double', ignore_errors=True)
+#long_double_size = rffi.sizeof_c_type('long double', ignore_errors=True)
+#import os
+#if long_double_size == 8 and os.name == 'nt':
+#    # this is a lie, or maybe a wish, MS fakes longdouble math with double
+#    long_double_size = 12
 
-import os
-if long_double_size == 8 and os.name == 'nt':
-    # this is a lie, or maybe a wish, MS fakes longdouble math with double
-    long_double_size = 12
+# hardcode to 8 for now (simulate using normal double) until long double works
+long_double_size = 8
 
 
 def new_dtype_getter(name):
@@ -438,32 +438,18 @@
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex128")
     _COMPONENTS_BOX = W_Float64Box
 
-if ENABLED_LONG_DOUBLE and long_double_size == 12:
-    class W_Float96Box(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float96")
 
-    W_LongDoubleBox = W_Float96Box
+if long_double_size == 8:
+    W_FloatLongBox = W_Float64Box
+    W_ComplexLongBox = W_Complex128Box
 
-    class W_Complex192Box(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex192")
-        _COMPONENTS_BOX = W_Float96Box
+elif long_double_size in (12, 16):
+    class W_FloatLongBox(W_FloatingBox, PrimitiveBox):
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % long_double_size * 8)
 
-    W_CLongDoubleBox = W_Complex192Box
-
-elif ENABLED_LONG_DOUBLE and long_double_size == 16:
-    class W_Float128Box(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float128")
-    W_LongDoubleBox = W_Float128Box
-
-    class W_Complex256Box(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex256")
-        _COMPONENTS_BOX = W_Float128Box
-
-    W_CLongDoubleBox = W_Complex256Box
-
-elif ENABLED_LONG_DOUBLE:
-    W_LongDoubleBox = W_Float64Box
-    W_CLongDoubleBox = W_Complex64Box
+    class W_ComplexLongBox(ComplexBox, W_ComplexFloatingBox):
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % long_double_size * 16)
+        _COMPONENTS_BOX = W_FloatLongBox
 
 
 W_GenericBox.typedef = TypeDef("generic",
@@ -647,34 +633,18 @@
     __reduce__ = interp2app(W_Float64Box.descr_reduce),
 )
 
-if ENABLED_LONG_DOUBLE and long_double_size == 12:
-    W_Float96Box.typedef = TypeDef("float96", (W_FloatingBox.typedef),
+if long_double_size in (12, 16):
+    W_FloatLongBox.typedef = TypeDef("float%d" % long_double_size * 8, (W_FloatingBox.typedef),
         __module__ = "numpypy",
-        __reduce__ = interp2app(W_Float96Box.descr_reduce),
+        __reduce__ = interp2app(W_FloatLongBox.descr_reduce),
 
-        __new__ = interp2app(W_Float96Box.descr__new__.im_func),
+        __new__ = interp2app(W_FloatLongBox.descr__new__.im_func),
     )
 
-    W_Complex192Box.typedef = TypeDef("complex192", (W_ComplexFloatingBox.typedef, complex_typedef),
+    W_ComplexLongBox.typedef = TypeDef("complex%d" % long_double_size * 16, (W_ComplexFloatingBox.typedef, complex_typedef),
         __module__ = "numpypy",
-        __new__ = interp2app(W_Complex192Box.descr__new__.im_func),
-        __reduce__ = interp2app(W_Complex192Box.descr_reduce),
-        real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
-        imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
-    )
-
-elif ENABLED_LONG_DOUBLE and long_double_size == 16:
-    W_Float128Box.typedef = TypeDef("float128", (W_FloatingBox.typedef),
-        __module__ = "numpypy",
-
-        __new__ = interp2app(W_Float128Box.descr__new__.im_func),
-        __reduce__ = interp2app(W_Float128Box.descr_reduce),
-    )
-
-    W_Complex256Box.typedef = TypeDef("complex256", (W_ComplexFloatingBox.typedef, complex_typedef),
-        __module__ = "numpypy",
-        __new__ = interp2app(W_Complex256Box.descr__new__.im_func),
-        __reduce__ = interp2app(W_Complex256Box.descr_reduce),
+        __new__ = interp2app(W_ComplexLongBox.descr__new__.im_func),
+        __reduce__ = interp2app(W_ComplexLongBox.descr_reduce),
         real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
         imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
     )
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -542,15 +542,11 @@
             char="I",
             w_box_type=space.gettypefor(interp_boxes.W_UInt32Box),
         )
-        if LONG_BIT == 32:
-            name = "int32"
-        elif LONG_BIT == 64:
-            name = "int64"
         self.w_longdtype = W_Dtype(
             types.Long(),
             num=7,
             kind=SIGNEDLTR,
-            name=name,
+            name="int%d" % LONG_BIT,
             char="l",
             w_box_type=space.gettypefor(interp_boxes.W_LongBox),
             alternate_constructors=[space.w_int,
@@ -563,7 +559,7 @@
             types.ULong(),
             num=8,
             kind=UNSIGNEDLTR,
-            name="u" + name,
+            name="uint%d" % LONG_BIT,
             char="L",
             w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
             alternate_constructors=[ space.gettypefor(interp_boxes.W_UnsignedIntegerBox),
@@ -607,6 +603,15 @@
                                    ],
             aliases=["float", "double"],
         )
+        self.w_floatlongdtype = W_Dtype(
+            types.FloatLong(),
+            num=13,
+            kind=FLOATINGLTR,
+            name="float%d" % interp_boxes.long_double_size * 8,
+            char="g",
+            w_box_type=space.gettypefor(interp_boxes.W_FloatLongBox),
+            aliases=["longdouble", "longfloat"],
+        )
         self.w_complex64dtype = W_ComplexDtype(
             types.Complex64(),
             num=14,
@@ -627,57 +632,16 @@
             aliases=["complex"],
             float_type = self.w_float64dtype,
         )
-        if interp_boxes.ENABLED_LONG_DOUBLE and interp_boxes.long_double_size == 12:
-            self.w_float96dtype = W_Dtype(
-                types.Float96(),
-                num=13,
-                kind=FLOATINGLTR,
-                name="float96",
-                char="g",
-                w_box_type=space.gettypefor(interp_boxes.W_Float96Box),
-                aliases=["longdouble", "longfloat"],
-            )
-            self.w_complex192dtype = W_ComplexDtype(
-                types.Complex192(),
-                num=16,
-                kind=COMPLEXLTR,
-                name="complex192",
-                char="G",
-                w_box_type = space.gettypefor(interp_boxes.W_Complex192Box),
-                alternate_constructors=[space.w_complex],
-                aliases=["clongdouble", "clongfloat"],
-                float_type = self.w_float96dtype,
-            )
-            self.w_longdouble = self.w_float96dtype
-            self.w_clongdouble = self.w_complex192dtype
-        elif interp_boxes.ENABLED_LONG_DOUBLE and interp_boxes.long_double_size == 16:
-            self.w_float128dtype = W_Dtype(
-                types.Float128(),
-                num=13,
-                kind=FLOATINGLTR,
-                name="float128",
-                char="g",
-                w_box_type=space.gettypefor(interp_boxes.W_Float128Box),
-                aliases=["longdouble", "longfloat"],
-            )
-            self.w_complex256dtype = W_ComplexDtype(
-                types.Complex256(),
-                num=16,
-                kind=COMPLEXLTR,
-                name="complex256",
-                char="G",
-                w_box_type = space.gettypefor(interp_boxes.W_Complex256Box),
-                alternate_constructors=[space.w_complex],
-                aliases=["clongdouble", "clongfloat"],
-                float_type = self.w_float128dtype,
-            )
-            self.w_longdouble = self.w_float128dtype
-            self.w_clongdouble = self.w_complex256dtype
-        elif interp_boxes.ENABLED_LONG_DOUBLE:
-            self.w_float64dtype.aliases += ["longdouble", "longfloat"]
-            self.w_complex128dtype.aliases += ["clongdouble", "clongfloat"]
-            self.w_longdouble = self.w_float64dtype
-            self.w_clongdouble = self.w_complex128dtype
+        self.w_complexlongdtype = W_ComplexDtype(
+            types.ComplexLong(),
+            num=16,
+            kind=COMPLEXLTR,
+            name="complex%d" % interp_boxes.long_double_size * 16,
+            char="G",
+            w_box_type = space.gettypefor(interp_boxes.W_ComplexLongBox),
+            aliases=["clongdouble", "clongfloat"],
+            float_type = self.w_floatlongdtype,
+        )
         self.w_stringdtype = W_Dtype(
             types.StringType(0),
             num=18,
@@ -750,21 +714,18 @@
             char=UINTPLTR,
             w_box_type = space.gettypefor(uintp_box),
         )
-        float_dtypes = [self.w_float16dtype,
-                self.w_float32dtype, self.w_float64dtype,
-                ]
-        complex_dtypes =  [self.w_complex64dtype, self.w_complex128dtype]
-        if interp_boxes.ENABLED_LONG_DOUBLE:
-            float_dtypes.append(self.w_longdouble)
-            complex_dtypes.append(self.w_clongdouble)
+        float_dtypes = [self.w_float16dtype, self.w_float32dtype,
+                        self.w_float64dtype, self.w_floatlongdtype]
+        complex_dtypes = [self.w_complex64dtype, self.w_complex128dtype,
+                          self.w_complexlongdtype]
         self.builtin_dtypes = [
             self.w_booldtype,
             self.w_int8dtype, self.w_uint8dtype,
             self.w_int16dtype, self.w_uint16dtype,
             self.w_longdtype, self.w_ulongdtype,
             self.w_int32dtype, self.w_uint32dtype,
-            self.w_int64dtype, self.w_uint64dtype] + \
-            float_dtypes + complex_dtypes + [
+            self.w_int64dtype, self.w_uint64dtype,
+            ] + float_dtypes + complex_dtypes + [
             self.w_stringdtype, self.w_unicodedtype, self.w_voiddtype,
             self.w_intpdtype, self.w_uintpdtype,
         ]
@@ -818,6 +779,7 @@
             'STRING': self.w_stringdtype,
             'CFLOAT': self.w_complex64dtype,
             'CDOUBLE': self.w_complex128dtype,
+            'CLONGDOUBLE': self.w_complexlongdtype,
             #'DATETIME',
             'UINT': self.w_uint32dtype,
             'INTP': self.w_intpdtype,
@@ -827,13 +789,11 @@
             #'TIMEDELTA',
             'INT': self.w_int32dtype,
             'DOUBLE': self.w_float64dtype,
+            'LONGDOUBLE': self.w_floatlongdtype,
             'USHORT': self.w_uint16dtype,
             'FLOAT': self.w_float32dtype,
             'BOOL': self.w_booldtype,
         }
-        if interp_boxes.ENABLED_LONG_DOUBLE:
-            typeinfo_full['LONGDOUBLE'] = self.w_longdouble
-            typeinfo_full['CLONGDOUBLE'] = self.w_clongdouble
 
         typeinfo_partial = {
             'Generic': interp_boxes.W_GenericBox,
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -449,8 +449,8 @@
             return interp_dtype.get_dtype_cache(space).w_complex64dtype
         elif dt2.num == 15:
             return interp_dtype.get_dtype_cache(space).w_complex128dtype
-        elif interp_boxes.ENABLED_LONG_DOUBLE and dt2.num == 16:
-            return interp_dtype.get_dtype_cache(space).w_clongdouble
+        elif dt2.num == 16:
+            return interp_dtype.get_dtype_cache(space).w_complexlongdtype
         else:
             raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
 
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -196,13 +196,8 @@
         raises(TypeError, signbit, complex(1,1))
 
     def test_reciprocal(self):
-        from numpypy import array, reciprocal, complex64, complex128
-        c_and_relerr = [(complex64, 2e-7), (complex128, 2e-15)]
-        try:
-            from numpypy import clongdouble
-            c_and_relerr.append((clongdouble, 2e-30))
-        except:
-            pass # no longdouble yet
+        from numpypy import array, reciprocal, complex64, complex128, clongdouble
+        c_and_relerr = [(complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)]
         inf = float('inf')
         nan = float('nan')
         #complex
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -914,36 +914,7 @@
         assert typeinfo['CDOUBLE'] == ('D', 15, 128, 8, complex128)
         assert typeinfo['HALF'] == ('e', 23, 16, 2, float16)
 
-class AppTestNoLongDoubleDtypes(BaseNumpyAppTest):
-    def setup_class(cls):
-        from pypy.module.micronumpy import Module
-        if Module.interpleveldefs.get('longfloat', None):
-            py.test.skip('longdouble exists, skip these tests')
-        if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("pypy only test for no longdouble support")
-        BaseNumpyAppTest.setup_class.im_func(cls)
-
-    def test_nolongfloat(self):
-        import numpypy
-        from numpypy import dtype
-        assert not getattr(numpypy, 'longdouble', False)
-        assert not getattr(numpypy, 'float128', False)
-        assert not getattr(numpypy, 'float96', False)
-        raises(TypeError, dtype, 'longdouble')
-        raises(TypeError, dtype, 'clongdouble')
-        raises(TypeError, dtype, 'longfloat')
-        raises(TypeError, dtype, 'clongfloat')
-        raises(TypeError, dtype, 'float128')
-        raises(TypeError, dtype, 'float96')
-
 class AppTestLongDoubleDtypes(BaseNumpyAppTest):
-    def setup_class(cls):
-        from pypy.module.micronumpy import Module
-        print dir(Module.interpleveldefs)
-        if not Module.interpleveldefs.get('longfloat', None):
-            py.test.skip('no longdouble types yet')
-        BaseNumpyAppTest.setup_class.im_func(cls)
-
     def test_longfloat(self):
         import numpypy as numpy
         # it can be float96 or float128
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1661,16 +1661,22 @@
 
 NonNativeComplex128 = Complex128
 
-if interp_boxes.ENABLED_LONG_DOUBLE and interp_boxes.long_double_size == 12:
-    class Float96(BaseType, Float):
+if interp_boxes.long_double_size == 8:
+    FloatLong = Float64
+    NonNativeFloatLong = NonNativeFloat64
+    ComplexLong = Complex128
+    NonNativeComplexLong = NonNativeComplex128
+
+elif interp_boxes.long_double_size in (12, 16):
+    class FloatLong(BaseType, Float):
         _attrs_ = ()
 
         T = rffi.LONGDOUBLE
-        BoxType = interp_boxes.W_Float96Box
+        BoxType = interp_boxes.W_FloatLongBox
         format_code = "q"
 
         def runpack_str(self, s):
-            assert len(s) == 12
+            assert len(s) == interp_boxes.long_double_size
             fval = unpack_float80(s, native_is_bigendian)
             return self.box(fval)
 
@@ -1680,46 +1686,17 @@
             pack_float80(result, value, 10, not native_is_bigendian)
             return self.box(unpack_float80(result.build(), native_is_bigendian))
 
-    NonNativeFloat96 = Float96
+    NonNativeFloatLong = FloatLong
 
-    class Complex192(ComplexFloating, BaseType):
+    class ComplexLong(ComplexFloating, BaseType):
         _attrs_ = ()
 
         T = rffi.LONGDOUBLE
-        BoxType = interp_boxes.W_Complex192Box
-        ComponentBoxType = interp_boxes.W_Float96Box
+        BoxType = interp_boxes.W_ComplexLongBox
+        ComponentBoxType = interp_boxes.W_FloatLongBox
 
-    NonNativeComplex192 = Complex192
+    NonNativeComplexLong = ComplexLong
 
-elif interp_boxes.ENABLED_LONG_DOUBLE and interp_boxes.long_double_size == 16:
-    class Float128(BaseType, Float):
-        _attrs_ = ()
-
-        T = rffi.LONGDOUBLE
-        BoxType = interp_boxes.W_Float128Box
-        format_code = "q"
-
-        def runpack_str(self, s):
-            assert len(s) == 16
-            fval = unpack_float80(s, native_is_bigendian)
-            return self.box(fval)
-
-        def byteswap(self, w_v):
-            value = self.unbox(w_v)
-            result = StringBuilder(10)
-            pack_float80(result, value, 10, not native_is_bigendian)
-            return self.box(unpack_float80(result.build(), native_is_bigendian))
-
-    NonNativeFloat128 = Float128
-
-    class Complex256(ComplexFloating, BaseType):
-        _attrs_ = ()
-
-        T = rffi.LONGDOUBLE
-        BoxType = interp_boxes.W_Complex256Box
-        ComponentBoxType = interp_boxes.W_Float128Box
-
-    NonNativeComplex256 = Complex256
 
 class BaseStringType(object):
     _mixin_ = True

From noreply at buildbot.pypy.org  Wed Oct 16 08:13:05 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 08:13:05 +0200 (CEST)
Subject: [pypy-commit] buildbot default: The "py.test -A" part of a
 pypy-c-jit build suddenly eats 20GB of memory
Message-ID: <20131016061305.70EE61C02C2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r867:ea1d86a3a8f5
Date: 2013-10-16 08:11 +0200
http://bitbucket.org/pypy/buildbot/changeset/ea1d86a3a8f5/

Log:	The "py.test -A" part of a pypy-c-jit build suddenly eats 20GB of
	memory on OS/X, apparently. Disabling the builder for now --- it's
	the only thing I can do.

diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -214,7 +214,8 @@
             JITFREEBSD764,             # on headless
             JITFREEBSD864,             # on ananke
             JITFREEBSD964,             # on exarkun's freebsd
-            JITMACOSX64,               # on xerxes
+            #JITMACOSX64,              # on xerxes
+            #^^^ status: "py.test -A" eats 20GB of memory apparently :-(
             # buildbot selftest
             PYPYBUILDBOT               # on cobra
             ], branch='default', hour=0, minute=0),

From noreply at buildbot.pypy.org  Wed Oct 16 08:13:06 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 08:13:06 +0200 (CEST)
Subject: [pypy-commit] buildbot default: The "py.test -A" part of a
 pypy-c-jit build suddenly eats 20GB of memory
Message-ID: <20131016061306.7D3C21C02C2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r868:df145500744c
Date: 2013-10-16 08:12 +0200
http://bitbucket.org/pypy/buildbot/changeset/df145500744c/

Log:	The "py.test -A" part of a pypy-c-jit build suddenly eats 20GB of
	memory on OS/X, apparently. Disabling the builder for now --- it's
	the only thing I can do.

diff --git a/bot2/pypybuildbot/arm_master.py b/bot2/pypybuildbot/arm_master.py
--- a/bot2/pypybuildbot/arm_master.py
+++ b/bot2/pypybuildbot/arm_master.py
@@ -147,6 +147,8 @@
         BUILDLINUXARM,                 # on hhu-cross-armel, uses 1 core
         BUILDLINUXARMHF_RASPBIAN,      # on hhu-cross-raspbianhf, uses 1 core
 
+        LINUXARMHF,                    # onw tests on greenbox3-node0
+
         JITBACKENDONLYLINUXARMEL,      # on hhu-imx.53
         JITBACKENDONLYLINUXARMHF,
         JITBACKENDONLYLINUXARMHF_v7,   # on cubieboard-bob
diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -167,6 +167,7 @@
 JITLINUX32 = "pypy-c-jit-linux-x86-32"
 JITLINUX64 = "pypy-c-jit-linux-x86-64"
 JITMACOSX64 = "pypy-c-jit-macosx-x86-64"
+JITMACOSX64_2 = "pypy-c-jit-macosx-x86-64-2"
 JITWIN32 = "pypy-c-jit-win-x86-32"
 JITWIN64 = "pypy-c-jit-win-x86-64"
 JITFREEBSD764 = 'pypy-c-jit-freebsd-7-x86-64'
@@ -253,6 +254,7 @@
                         JITLINUX32,
                         JITLINUX64,
                         JITMACOSX64,
+                        JITMACOSX64_2,
                         JITWIN32,
                         JITWIN64,
                         JITFREEBSD764,
@@ -353,11 +355,17 @@
                    "category": 'mac32'
                   },
                   {"name" : JITMACOSX64,
-                   "slavenames": ["xerxes"],
+                   "slavenames": ["xerxes", "tosh"],
                    'builddir' : JITMACOSX64,
                    'factory' : pypyJITTranslatedTestFactoryOSX64,
                    'category' : 'mac64',
                    },
+                  {"name" : JITMACOSX64_2,
+                   "slavenames": ["xerxes", "tosh"],
+                   'builddir' : JITMACOSX64_2,
+                   'factory' : pypyJITTranslatedTestFactoryOSX64,
+                   'category' : 'mac64',
+                   },
                   {"name": WIN32,
                    "slavenames": ["aurora", "SalsaSalsa"],
                    "builddir": WIN32,

From noreply at buildbot.pypy.org  Wed Oct 16 10:50:53 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 10:50:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: 32-bit support
Message-ID: <20131016085054.007421C02C2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67407:8eb4474573c2
Date: 2013-10-16 10:50 +0200
http://bitbucket.org/pypy/pypy/changeset/8eb4474573c2/

Log:	32-bit support

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -119,10 +119,15 @@
             typeids_txt = os.path.join(newroot, 'typeids.txt')
         print 'loading', typeids_txt
         typeids = {}
+        if sys.maxint < 2**32:
+            TIDT = "int*"
+        else:
+            TIDT = "char*"
         with open(typeids_txt) as f:
             for line in f:
                 member, descr = map(str.strip, line.split(None, 1))
-                expr = "((char*)(&pypy_g_typeinfo.%s)) - (char*)&pypy_g_typeinfo" % member
+                expr = ("((%s)(&pypy_g_typeinfo.%s)) - (%s)&pypy_g_typeinfo"
+                           % (TIDT, member, TIDT))
                 offset = int(self.gdb.parse_and_eval(expr))
                 typeids[offset] = descr
         return typeids
diff --git a/pypy/tool/test/test_gdb_pypy.py b/pypy/tool/test/test_gdb_pypy.py
--- a/pypy/tool/test/test_gdb_pypy.py
+++ b/pypy/tool/test/test_gdb_pypy.py
@@ -1,6 +1,11 @@
-import py
+import py, sys
 from pypy.tool import gdb_pypy
 
+if sys.maxint < 2**32:
+    TIDT = "int*"
+else:
+    TIDT = "char*"
+
 class FakeGdb(object):
 
     COMMAND_NONE = -1
@@ -108,7 +113,8 @@
 """.strip())
     progspace = Mock(filename=str(exe))
     exprs = {
-        '((char*)(&pypy_g_typeinfo.member0)) - (char*)&pypy_g_typeinfo': 0,
+        '((%s)(&pypy_g_typeinfo.member0)) - (%s)&pypy_g_typeinfo'
+            % (TIDT, TIDT): 0,
         }
     gdb = FakeGdb(exprs, progspace)
     cmd = gdb_pypy.RPyType(gdb)
@@ -135,9 +141,12 @@
     myvar = Value(d)
     exprs = {
         '*myvar': myvar,
-        '((char*)(&pypy_g_typeinfo.member0)) - (char*)&pypy_g_typeinfo': 0,
-        '((char*)(&pypy_g_typeinfo.member1)) - (char*)&pypy_g_typeinfo': 123,
-        '((char*)(&pypy_g_typeinfo.member2)) - (char*)&pypy_g_typeinfo': 456,
+        '((%s)(&pypy_g_typeinfo.member0)) - (%s)&pypy_g_typeinfo'
+            % (TIDT, TIDT): 0,
+        '((%s)(&pypy_g_typeinfo.member1)) - (%s)&pypy_g_typeinfo'
+            % (TIDT, TIDT): 123,
+        '((%s)(&pypy_g_typeinfo.member2)) - (%s)&pypy_g_typeinfo'
+            % (TIDT, TIDT): 456,
         }
     gdb = FakeGdb(exprs, progspace)
     cmd = gdb_pypy.RPyType(gdb)

From noreply at buildbot.pypy.org  Wed Oct 16 11:02:58 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 16 Oct 2013 11:02:58 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Add my talk at paris.py,
	in French :)
Message-ID: <20131016090258.EFBAA1D22F6@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: extradoc
Changeset: r5093:c6b907f56bb2
Date: 2013-10-16 11:02 +0200
http://bitbucket.org/pypy/extradoc/changeset/c6b907f56bb2/

Log:	Add my talk at paris.py, in French :)

diff --git a/talk/paris.py-3/Makefile b/talk/paris.py-3/Makefile
new file mode 100644
--- /dev/null
+++ b/talk/paris.py-3/Makefile
@@ -0,0 +1,16 @@
+# you can find rst2beamer.py and inkscapeslide.py here:
+# http://bitbucket.org/antocuni/env/src/619f486c4fad/bin/rst2beamer.py
+# http://bitbucket.org/antocuni/env/src/619f486c4fad/bin/inkscapeslide.py
+
+
+talk.pdf: talk.rst author.latex stylesheet.latex
+	rst2beamer.py --input-encoding=utf8 --output-encoding=utf8 --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
+	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit
+	pdflatex talk.latex  || exit
+
+view: talk.pdf
+	evince talk.pdf > /dev/null 2>&1 &
+
+xpdf: talk.pdf
+	xpdf talk.pdf &
diff --git a/talk/paris.py-3/Speed.png b/talk/paris.py-3/Speed.png
new file mode 100644
index 0000000000000000000000000000000000000000..796a1ed2ef8f48d701a54242e78694ac16a70762
GIT binary patch

[cut]

diff --git a/talk/paris.py-3/Time.png b/talk/paris.py-3/Time.png
new file mode 100644
index 0000000000000000000000000000000000000000..3618f1e4a4f55d3604a686051fce2f7fe31a3bda
GIT binary patch

[cut]

diff --git a/talk/paris.py-3/author.latex b/talk/paris.py-3/author.latex
new file mode 100644
--- /dev/null
+++ b/talk/paris.py-3/author.latex
@@ -0,0 +1,8 @@
+\definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0}
+
+\title[PyPy : l'impl&#233;mentation la plus rapide de Python]{PyPy : l'impl&#233;mentation la plus rapide de Python}
+\author[rguillebert]
+{Romain Guillebert}
+
+\institute{Paris.py}
+\date{15 octobre 2013}
diff --git a/talk/paris.py-3/beamerdefs.txt b/talk/paris.py-3/beamerdefs.txt
new file mode 100644
--- /dev/null
+++ b/talk/paris.py-3/beamerdefs.txt
@@ -0,0 +1,108 @@
+.. colors
+.. ===========================
+
+.. role:: green
+.. role:: red
+
+
+.. general useful commands
+.. ===========================
+
+.. |pause| raw:: latex
+
+   \pause
+
+.. |small| raw:: latex
+
+   {\small
+
+.. |end_small| raw:: latex
+
+   }
+
+.. |scriptsize| raw:: latex
+
+   {\scriptsize
+
+.. |end_scriptsize| raw:: latex
+
+   }
+
+.. |strike<| raw:: latex
+
+   \sout{
+
+.. closed bracket
+.. ===========================
+
+.. |>| raw:: latex
+
+   }
+
+
+.. example block
+.. ===========================
+
+.. |example<| raw:: latex
+
+   \begin{exampleblock}{
+
+
+.. |end_example| raw:: latex
+
+   \end{exampleblock}
+
+
+
+.. alert block
+.. ===========================
+
+.. |alert<| raw:: latex
+
+   \begin{alertblock}{
+
+
+.. |end_alert| raw:: latex
+
+   \end{alertblock}
+
+
+
+.. columns
+.. ===========================
+
+.. |column1| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.45\textwidth}
+
+.. |column2| raw:: latex
+
+      \end{column}
+      \begin{column}{0.45\textwidth}
+
+
+.. |end_columns| raw:: latex
+
+      \end{column}
+   \end{columns}
+
+
+
+.. |snake| image:: ../../img/py-web-new.png
+           :scale: 15%
+           
+
+
+.. nested blocks
+.. ===========================
+
+.. |nested| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.85\textwidth}
+
+.. |end_nested| raw:: latex
+
+      \end{column}
+   \end{columns}
diff --git a/talk/paris.py-3/stylesheet.latex b/talk/paris.py-3/stylesheet.latex
new file mode 100644
--- /dev/null
+++ b/talk/paris.py-3/stylesheet.latex
@@ -0,0 +1,11 @@
+\usetheme{Boadilla}
+\usecolortheme{whale}
+\setbeamercovered{transparent}
+\setbeamertemplate{navigation symbols}{}
+
+\definecolor{darkgreen}{rgb}{0, 0.5, 0.0}
+\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor}
+
+\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\red}[1]{\color{red}#1\normalcolor}
diff --git a/talk/paris.py-3/talk.pdf b/talk/paris.py-3/talk.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..70e6beb3cd10de4a2fa5cadc8402aafb095efad0
GIT binary patch

[cut]

diff --git a/talk/paris.py-3/talk.rst b/talk/paris.py-3/talk.rst
new file mode 100644
--- /dev/null
+++ b/talk/paris.py-3/talk.rst
@@ -0,0 +1,126 @@
+.. include:: beamerdefs.txt
+
+================================================
+PyPy : l'impl&#233;mentation la plus rapide de Python
+================================================
+
+Terminologie
+------------
+
+* Python
+
+* CPython
+
+* PyPy
+
+* RPython
+
+Introduction
+------------
+
+* PyPy est un interpreteur Python &#233;crit en RPython qui vise &#224; &#234;tre le plus rapide possible
+
+* Vise &#224; ex&#233;cuter n'importe quel code Python
+
+RPython
+-------
+
+* Langage/Framework pour &#233;crire des machines virtuelles
+
+* Sous-ensemble de Python
+
+* Vivement d&#233;conseill&#233; pour d'autres usages
+
+|pause|
+
+* Ajoute (quasi) automatiquement un compilateur juste-a-temps et un ramasse miette (stop-the-world ou incr&#233;mental)
+
+* Nombreux langages (plus ou moins) impl&#233;ment&#233;s avec ce Framework : Python (PyPy), Ruby (Topaz), PHP(Hippy), Scheme, Smalltalk, Emulateur GameBoy, Brainfuck
+
+PyPy
+----
+
+* Vieux de plus de 10 ans
+
+* Cr&#233;er par (entre autres) Armin Rigo comme un rempla&#231;ant de psyco
+
+* Supporte x86, x86_64, ARM
+
+* Supporte Linux, Mac OS X, et Windows 32 bits
+
+* Production ready&#8482;
+
+speed.pypy.org
+--------------
+
+.. image:: Speed.png
+   :scale: 40%
+   :align: center
+
+speed.pypy.org
+--------------
+
+.. image:: Time.png
+   :scale: 40%
+   :align: center
+
+Demo
+----
+
+* `sobel.py`
+
+JIT
+---
+
+* Optimise les boucles et les fonctions recursives
+
+* Meta-Tracing JIT
+
+* Trace une boucle &#224; la 1039i&#232;me it&#233;ration
+
+* Inline presque tout
+
+* Compile un seul chemin a la fois
+
+Compatibilit&#233;
+-------------
+
+* 100% compatible avec la sp&#233;cification du langage
+
+* Attention aux destructeurs (fermez vos fichiers)
+
+* Support des extensions C en version b&#233;ta, performances m&#233;diocres
+
+Futur
+-----
+
+* STM
+
+* Numpy
+
+* Python 3
+
+* Appel aux dons sur ces 3 projets : pypy.org
+
+Numpy
+-----
+
+* R&#233;&#233;criture du code C en RPython (jit-friendly)
+
+* R&#233;utilisation du code Python
+
+|pause|
+
+* Compatibilit&#233;
+
+* Vitesse
+
+Demo de Numpy
+-------------
+
+* Demo ndarray() + ndarray() VS it&#233;ration
+
+Questions ?
+-----------
+
+Questions ?

From noreply at buildbot.pypy.org  Wed Oct 16 12:13:21 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 12:13:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: typos/cleanups
Message-ID: <20131016101321.BF9661C0203@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67408:394a633e8c93
Date: 2013-10-16 05:27 -0400
http://bitbucket.org/pypy/pypy/changeset/394a633e8c93/

Log:	typos/cleanups

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -445,10 +445,10 @@
 
 elif long_double_size in (12, 16):
     class W_FloatLongBox(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % long_double_size * 8)
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % (long_double_size * 8))
 
     class W_ComplexLongBox(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % long_double_size * 16)
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % (long_double_size * 16))
         _COMPONENTS_BOX = W_FloatLongBox
 
 
@@ -614,34 +614,50 @@
 
 W_Float16Box.typedef = TypeDef("float16", W_FloatingBox.typedef,
     __module__ = "numpypy",
-
     __new__ = interp2app(W_Float16Box.descr__new__.im_func),
     __reduce__ = interp2app(W_Float16Box.descr_reduce),
 )
 
 W_Float32Box.typedef = TypeDef("float32", W_FloatingBox.typedef,
     __module__ = "numpypy",
-
     __new__ = interp2app(W_Float32Box.descr__new__.im_func),
     __reduce__ = interp2app(W_Float32Box.descr_reduce),
 )
 
 W_Float64Box.typedef = TypeDef("float64", (W_FloatingBox.typedef, float_typedef),
     __module__ = "numpypy",
-
     __new__ = interp2app(W_Float64Box.descr__new__.im_func),
     __reduce__ = interp2app(W_Float64Box.descr_reduce),
 )
 
+W_ComplexFloatingBox.typedef = TypeDef("complexfloating", W_InexactBox.typedef,
+    __module__ = "numpypy",
+)
+
+W_Complex64Box.typedef = TypeDef("complex64", (W_ComplexFloatingBox.typedef),
+    __module__ = "numpypy",
+    __new__ = interp2app(W_Complex64Box.descr__new__.im_func),
+    __reduce__ = interp2app(W_Complex64Box.descr_reduce),
+    real = GetSetProperty(W_ComplexFloatingBox .descr_get_real),
+    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
+)
+
+W_Complex128Box.typedef = TypeDef("complex128", (W_ComplexFloatingBox.typedef, complex_typedef),
+    __module__ = "numpypy",
+    __new__ = interp2app(W_Complex128Box.descr__new__.im_func),
+    __reduce__ = interp2app(W_Complex128Box.descr_reduce),
+    real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
+    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
+)
+
 if long_double_size in (12, 16):
-    W_FloatLongBox.typedef = TypeDef("float%d" % long_double_size * 8, (W_FloatingBox.typedef),
+    W_FloatLongBox.typedef = TypeDef("float%d" % (long_double_size * 8), (W_FloatingBox.typedef),
         __module__ = "numpypy",
+        __new__ = interp2app(W_FloatLongBox.descr__new__.im_func),
         __reduce__ = interp2app(W_FloatLongBox.descr_reduce),
-
-        __new__ = interp2app(W_FloatLongBox.descr__new__.im_func),
     )
 
-    W_ComplexLongBox.typedef = TypeDef("complex%d" % long_double_size * 16, (W_ComplexFloatingBox.typedef, complex_typedef),
+    W_ComplexLongBox.typedef = TypeDef("complex%d" % (long_double_size * 16), (W_ComplexFloatingBox.typedef, complex_typedef),
         __module__ = "numpypy",
         __new__ = interp2app(W_ComplexLongBox.descr__new__.im_func),
         __reduce__ = interp2app(W_ComplexLongBox.descr_reduce),
@@ -673,24 +689,3 @@
     __module__ = "numpypy",
     __new__ = interp2app(W_UnicodeBox.descr__new__unicode_box.im_func),
 )
-
-W_ComplexFloatingBox.typedef = TypeDef("complexfloating", W_InexactBox.typedef,
-    __module__ = "numpypy",
-)
-
-
-W_Complex128Box.typedef = TypeDef("complex128", (W_ComplexFloatingBox.typedef, complex_typedef),
-    __module__ = "numpypy",
-    __new__ = interp2app(W_Complex128Box.descr__new__.im_func),
-    __reduce__ = interp2app(W_Complex128Box.descr_reduce),
-    real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
-    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
-)
-
-W_Complex64Box.typedef = TypeDef("complex64", (W_ComplexFloatingBox.typedef),
-    __module__ = "numpypy",
-    __new__ = interp2app(W_Complex64Box.descr__new__.im_func),
-    __reduce__ = interp2app(W_Complex64Box.descr_reduce),
-    real = GetSetProperty(W_ComplexFloatingBox .descr_get_real),
-    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
-)
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -607,7 +607,7 @@
             types.FloatLong(),
             num=13,
             kind=FLOATINGLTR,
-            name="float%d" % interp_boxes.long_double_size * 8,
+            name="float%d" % (interp_boxes.long_double_size * 8),
             char="g",
             w_box_type=space.gettypefor(interp_boxes.W_FloatLongBox),
             aliases=["longdouble", "longfloat"],
@@ -636,7 +636,7 @@
             types.ComplexLong(),
             num=16,
             kind=COMPLEXLTR,
-            name="complex%d" % interp_boxes.long_double_size * 16,
+            name="complex%d" % (interp_boxes.long_double_size * 16),
             char="G",
             w_box_type = space.gettypefor(interp_boxes.W_ComplexLongBox),
             aliases=["clongdouble", "clongfloat"],
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1673,7 +1673,6 @@
 
         T = rffi.LONGDOUBLE
         BoxType = interp_boxes.W_FloatLongBox
-        format_code = "q"
 
         def runpack_str(self, s):
             assert len(s) == interp_boxes.long_double_size

From noreply at buildbot.pypy.org  Wed Oct 16 12:13:23 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 12:13:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: cleanups for tests
Message-ID: <20131016101323.114491C0203@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67409:02e7f1444c7b
Date: 2013-10-16 04:58 -0400
http://bitbucket.org/pypy/pypy/changeset/02e7f1444c7b/

Log:	cleanups for tests

diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -197,7 +197,6 @@
 
     def test_reciprocal(self):
         from numpypy import array, reciprocal, complex64, complex128, clongdouble
-        c_and_relerr = [(complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)]
         inf = float('inf')
         nan = float('nan')
         #complex
@@ -212,7 +211,7 @@
                     complex(-r, i),
                     -0j, 0j, cnan,
                     cnan, cnan, cnan]
-        for c, rel_err in c_and_relerr:
+        for c, rel_err in ((complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)):
             actual = reciprocal(array([orig], dtype=c))
             for b, a, e in zip(orig, actual, expected):
                 assert (a[0].real - e.real) < rel_err
@@ -232,18 +231,12 @@
         raises(TypeError, copysign, a, b)
 
     def test_exp2(self):
-        from numpypy import array, exp2, complex128, complex64
-        c_and_relerr = [(complex64, 2e-7), (complex128, 2e-15)]
-        try:
-            from numpypy import clongdouble
-            c_and_relerr.append((clongdouble, 2e-30))
-        except:
-            pass # no longdouble yet
+        from numpypy import array, exp2, complex128, complex64, clongdouble
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
         cmpl = complex
-        for c,rel_err in c_and_relerr:
+        for c, rel_err in ((complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)):
             a = [cmpl(-5., 0), cmpl(-5., -5.), cmpl(-5., 5.),
                        cmpl(0., -5.), cmpl(0., 0.), cmpl(0., 5.),
                        cmpl(-0., -5.), cmpl(-0., 0.), cmpl(-0., 5.),
@@ -274,12 +267,12 @@
 
     def test_expm1(self):
         import math, cmath
-        from numpypy import array, expm1, complex128, complex64
+        from numpypy import array, expm1, complex128, complex64, clongdouble
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
         cmpl = complex
-        for c,rel_err in ((complex128, 2e-15), (complex64, 1e-7)):
+        for c, rel_err in ((complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)):
             a = [cmpl(-5., 0), cmpl(-5., -5.), cmpl(-5., 5.),
                        cmpl(0., -5.), cmpl(0., 0.), cmpl(0., 5.),
                        cmpl(-0., -5.), cmpl(-0., 0.), cmpl(-0., 5.),
@@ -503,15 +496,9 @@
     def test_basic(self):
         from numpypy import (complex128, complex64, add, array, dtype,
             subtract as sub, multiply, divide, negative, absolute as abs,
-            floor_divide, real, imag, sign)
+            floor_divide, real, imag, sign, clongdouble)
         from numpypy import (equal, not_equal, greater, greater_equal, less,
                 less_equal, isnan)
-        complex_dtypes = [complex64, complex128]
-        try:
-            from numpypy import clongfloat
-            complex_dtypes.append(clongfloat)
-        except:
-            pass
         assert real(4.0) == 4.0
         assert imag(0.0) == 0.0
         a = array([complex(3.0, 4.0)])
@@ -540,8 +527,7 @@
         assert str(a.real) == 'abc'
         # numpy imag for flexible types returns self
         assert str(a.imag) == 'abc'
-        for complex_ in complex_dtypes:
-
+        for complex_ in complex64, complex128, clongdouble:
             O = complex(0, 0)
             c0 = complex_(complex(2.5, 0))
             c1 = complex_(complex(1, 2))
@@ -572,7 +558,6 @@
             assert negative(complex(1,1)) == complex(-1, -1)
             assert negative(complex(0, 0)) == 0
 
-
             assert multiply(1, c1) == c1
             assert multiply(2, c2) == complex(6, 8)
             assert multiply(c1, c2) == complex(-5, 10)
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -45,6 +45,13 @@
         raises(TypeError, lambda: dtype("int8") == 3)
         assert dtype(bool) == bool
 
+    def test_dtype_aliases(self):
+        from numpypy import dtype
+        assert dtype('longfloat').num in (12, 13)
+        assert dtype('longdouble').num in (12, 13)
+        assert dtype('clongfloat').num in (15, 16)
+        assert dtype('clongdouble').num in (15, 16)
+
     def test_dtype_with_types(self):
         from numpypy import dtype
 
@@ -153,6 +160,8 @@
             '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd',
             'e'
         ]
+        if array([0], dtype='longdouble').itemsize > 8:
+            types += ['g', 'G']
         a = array([True], '?')
         for t in types:
             assert (a + array([0], t)).dtype is dtype(t)
@@ -268,6 +277,7 @@
             (numpy.float16, 10.),
             (numpy.float32, 2.0),
             (numpy.float64, 4.32),
+            (numpy.longdouble, 4.32),
         ]:
             assert hash(tp(value)) == hash(value)
 
@@ -533,6 +543,20 @@
         from math import isnan
         assert isnan(numpy.float32(None))
         assert isnan(numpy.float64(None))
+        assert isnan(numpy.longdouble(None))
+
+    def test_longfloat(self):
+        import numpypy as numpy
+        # it can be float96 or float128
+        if numpy.longfloat != numpy.float64:
+            assert numpy.longfloat.mro()[1:] == [numpy.floating,
+                                       numpy.inexact, numpy.number,
+                                       numpy.generic, object]
+        a = numpy.array([1, 2, 3], numpy.longdouble)
+        assert type(a[1]) is numpy.longdouble
+        assert numpy.float64(12) == numpy.longdouble(12)
+        assert numpy.float64(12) == numpy.longfloat(12)
+        raises(ValueError, numpy.longfloat, '23.2df')
 
     def test_complex_floating(self):
         import numpypy as numpy
@@ -896,6 +920,12 @@
         a = array([1, 2, 3], dtype=self.non_native_prefix + 'f2')
         assert a[0] == 1
         assert (a + a)[1] == 4
+        a = array([1, 2, 3], dtype=self.non_native_prefix + 'g') # longdouble
+        assert a[0] == 1
+        assert (a + a)[1] == 4
+        a = array([1, 2, 3], dtype=self.non_native_prefix + 'G') # clongdouble
+        assert a[0] == 1
+        assert (a + a)[1] == 4
 
 class AppTestPyPyOnly(BaseNumpyAppTest):
     def setup_class(cls):
@@ -914,55 +944,6 @@
         assert typeinfo['CDOUBLE'] == ('D', 15, 128, 8, complex128)
         assert typeinfo['HALF'] == ('e', 23, 16, 2, float16)
 
-class AppTestLongDoubleDtypes(BaseNumpyAppTest):
-    def test_longfloat(self):
-        import numpypy as numpy
-        # it can be float96 or float128
-        if numpy.longfloat != numpy.float64:
-            assert numpy.longfloat.mro()[1:] == [numpy.floating,
-                                       numpy.inexact, numpy.number,
-                                       numpy.generic, object]
-        a = numpy.array([1, 2, 3], numpy.longdouble)
-        assert type(a[1]) is numpy.longdouble
-        assert numpy.float64(12) == numpy.longdouble(12)
-        assert numpy.float64(12) == numpy.longfloat(12)
-        raises(ValueError, numpy.longfloat, '23.2df')
-
-    def test_dtype_aliases(self):
-        from numpypy import dtype
-        assert dtype('longfloat').num in (12, 13)
-        assert dtype('longdouble').num in (12, 13)
-        assert dtype('clongfloat').num in (15, 16)
-        assert dtype('clongdouble').num in (15, 16)
-
-    def test_bool_binop_types(self):
-        from numpypy import array, dtype
-        types = ['g', 'G']
-        a = array([True], '?')
-        for t in types:
-            assert (a + array([0], t)).dtype is dtype(t)
-
-    def test_hash(self):
-        import numpypy as numpy
-        for tp, value in [
-            (numpy.longdouble, 4.32),
-        ]:
-            assert hash(tp(value)) == hash(value)
-
-    def test_float_None(self):
-        import numpypy as numpy
-        from math import isnan
-        assert isnan(numpy.longdouble(None))
-
-    def test_non_native(self):
-        from numpypy import array
-        a = array([1, 2, 3], dtype=self.non_native_prefix + 'g') # longdouble
-        assert a[0] == 1
-        assert (a + a)[1] == 4
-        a = array([1, 2, 3], dtype=self.non_native_prefix + 'G') # clongdouble
-        assert a[0] == 1
-        assert (a + a)[1] == 4
-
 class AppTestObjectDtypes(BaseNumpyAppTest):
     def test_scalar_from_object(self):
         from numpypy import array
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -365,6 +365,26 @@
         assert b[0] == 1+0j
         assert b.dtype is dtype(complex)
 
+    def test_arange(self):
+        from numpypy import arange, dtype
+        a = arange(3)
+        assert (a == [0, 1, 2]).all()
+        assert a.dtype is dtype(int)
+        a = arange(3.0)
+        assert (a == [0., 1., 2.]).all()
+        assert a.dtype is dtype(float)
+        a = arange(3, 7)
+        assert (a == [3, 4, 5, 6]).all()
+        assert a.dtype is dtype(int)
+        a = arange(3, 7, 2)
+        assert (a == [3, 5]).all()
+        a = arange(3, dtype=float)
+        assert (a == [0., 1., 2.]).all()
+        assert a.dtype is dtype(float)
+        a = arange(0, 0.8, 0.1)
+        assert len(a) == 8
+        assert arange(False, True, True).dtype is dtype(int)
+
     def test_copy(self):
         from numpypy import arange, array
         a = arange(5)
@@ -430,24 +450,17 @@
 
     def test_getitem_obj_index(self):
         from numpypy import arange
-
         a = arange(10)
-
         assert a[self.CustomIndexObject(1)] == 1
 
     def test_getitem_obj_prefer_index_to_int(self):
         from numpypy import arange
-
         a = arange(10)
-
-
         assert a[self.CustomIndexIntObject(0, 1)] == 0
 
     def test_getitem_obj_int(self):
         from numpypy import arange
-
         a = arange(10)
-
         assert a[self.CustomIntObject(1)] == 1
 
     def test_setitem(self):
@@ -469,7 +482,6 @@
         assert a[1] == -0.005
         assert a[2] == -0.005
 
-
     def test_setitem_tuple(self):
         from numpypy import array
         a = array(range(5))
@@ -483,27 +495,20 @@
 
     def test_setitem_obj_index(self):
         from numpypy import arange
-
         a = arange(10)
-
         a[self.CustomIndexObject(1)] = 100
         assert a[1] == 100
 
     def test_setitem_obj_prefer_index_to_int(self):
         from numpypy import arange
-
         a = arange(10)
-
         a[self.CustomIndexIntObject(0, 1)] = 100
         assert a[0] == 100
 
     def test_setitem_obj_int(self):
         from numpypy import arange
-
         a = arange(10)
-
         a[self.CustomIntObject(1)] = 100
-
         assert a[1] == 100
 
     def test_access_swallow_exception(self):
@@ -1872,6 +1877,15 @@
             i2 = (i+1) * a.dtype.itemsize
             assert list(reversed(s1[i1:i2])) == s2[i1:i2]
 
+        a = array([1, -1, 10000], dtype='longfloat')
+        s1 = map(ord, a.tostring())
+        s2 = map(ord, a.byteswap().tostring())
+        assert a.dtype.itemsize >= 8
+        for i in range(a.size):
+            i1 = i * a.dtype.itemsize
+            i2 = (i+1) * a.dtype.itemsize
+            assert list(reversed(s1[i1:i2])) == s2[i1:i2]
+
     def test_clip(self):
         from numpypy import array
         a = array([1, 2, 17, -3, 12])
@@ -2647,7 +2661,7 @@
 
     def test_fromstring_types(self):
         from numpypy import (fromstring, int8, int16, int32, int64, uint8,
-            uint16, uint32, float16, float32, float64, array)
+            uint16, uint32, float16, float32, float64, longfloat, array)
         a = fromstring('\xFF', dtype=int8)
         assert a[0] == -1
         b = fromstring('\xFF', dtype=uint8)
@@ -2670,6 +2684,18 @@
         assert j[0] == 12
         k = fromstring(self.float16val, dtype=float16)
         assert k[0] == float16(5.)
+        dt =  array([5],dtype=longfloat).dtype
+        if dt.itemsize == 12:
+            from numpypy import float96
+            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00', dtype=float96)
+        elif dt.itemsize == 16:
+            from numpypy import float128
+            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00\x00\x00\x00\x00', dtype=float128)
+        elif dt.itemsize == 8:
+            skip('longfloat is float64')
+        else:
+            skip('unknown itemsize for longfloat')
+        assert m[0] == longfloat(5.)
 
     def test_fromstring_invalid(self):
         from numpypy import fromstring, uint16, uint8
@@ -2689,28 +2715,6 @@
         assert array(0, dtype='i2').tostring() == '\x00\x00'
 
 
-class AppTestRanges(BaseNumpyAppTest):
-    def test_arange(self):
-        from numpypy import arange, dtype
-        a = arange(3)
-        assert (a == [0, 1, 2]).all()
-        assert a.dtype is dtype(int)
-        a = arange(3.0)
-        assert (a == [0., 1., 2.]).all()
-        assert a.dtype is dtype(float)
-        a = arange(3, 7)
-        assert (a == [3, 4, 5, 6]).all()
-        assert a.dtype is dtype(int)
-        a = arange(3, 7, 2)
-        assert (a == [3, 5]).all()
-        a = arange(3, dtype=float)
-        assert (a == [0., 1., 2.]).all()
-        assert a.dtype is dtype(float)
-        a = arange(0, 0.8, 0.1)
-        assert len(a) == 8
-        assert arange(False, True, True).dtype is dtype(int)
-
-
 class AppTestRepr(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect:
@@ -3027,40 +3031,3 @@
         assert x.__pypy_data__ is obj
         del x.__pypy_data__
         assert x.__pypy_data__ is None
-
-class AppTestLongDoubleDtypes(BaseNumpyAppTest):
-    def setup_class(cls):
-        from pypy.module.micronumpy import Module
-        #print dir(Module.interpleveldefs)
-        if not Module.interpleveldefs.get('longfloat', None):
-            py.test.skip('no longdouble types yet')
-        BaseNumpyAppTest.setup_class.im_func(cls)
-
-    def test_byteswap(self):
-        from numpypy import array
-
-        a = array([1, -1, 10000], dtype='longfloat')
-        s1 = map(ord, a.tostring())
-        s2 = map(ord, a.byteswap().tostring())
-        assert a.dtype.itemsize >= 8
-        for i in range(a.size):
-            i1 = i * a.dtype.itemsize
-            i2 = (i+1) * a.dtype.itemsize
-            assert list(reversed(s1[i1:i2])) == s2[i1:i2]
-
-    def test_fromstring_types(self):
-        from numpypy import (fromstring, longfloat, array)
-        dt =  array([5],dtype=longfloat).dtype
-        if dt.itemsize == 12:
-            from numpypy import float96
-            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00', dtype=float96)
-        elif dt.itemsize==16:
-            from numpypy import float128
-            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00\x00\x00\x00\x00', dtype=float128)
-        elif dt.itemsize == 8:
-            skip('longfloat is float64')
-        else:
-            skip('unknown itemsize for longfloat')
-        assert m[0] == longfloat(5.)
-
-

From noreply at buildbot.pypy.org  Wed Oct 16 12:13:24 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 12:13:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: try to enable numpypy long double
Message-ID: <20131016101324.43F7D1C0203@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67410:f694acf8afd7
Date: 2013-10-16 05:20 -0400
http://bitbucket.org/pypy/pypy/changeset/f694acf8afd7/

Log:	try to enable numpypy long double

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -20,14 +20,11 @@
 MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
 MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else ()
 
-#long_double_size = rffi.sizeof_c_type('long double', ignore_errors=True)
-#import os
-#if long_double_size == 8 and os.name == 'nt':
-#    # this is a lie, or maybe a wish, MS fakes longdouble math with double
-#    long_double_size = 12
-
-# hardcode to 8 for now (simulate using normal double) until long double works
-long_double_size = 8
+long_double_size = rffi.sizeof_c_type('long double', ignore_errors=True)
+import os
+if long_double_size == 8 and os.name == 'nt':
+    # this is a lie, or maybe a wish, MS fakes longdouble math with double
+    long_double_size = 12
 
 
 def new_dtype_getter(name):

From noreply at buildbot.pypy.org  Wed Oct 16 13:20:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 13:20:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: Hack hack hack at gdb_pypy: no longer
 preload all of typeids.txt, 
Message-ID: <20131016112035.718F21C0203@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67411:c9c7dba9ecc0
Date: 2013-10-16 13:19 +0200
http://bitbucket.org/pypy/pypy/changeset/c9c7dba9ecc0/

Log:	Hack hack hack at gdb_pypy: no longer preload all of typeids.txt,
	which takes forever.

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -113,24 +113,68 @@
         """
         exename = progspace.filename
         root = os.path.dirname(exename)
+        # XXX The same information is found in
+        # XXX   pypy_g_rpython_memory_gctypelayout_GCData.gcd_inst_typeids_z
+        # XXX Find out how to read it
         typeids_txt = os.path.join(root, 'typeids.txt')
         if not os.path.exists(typeids_txt):
             newroot = os.path.dirname(root)
             typeids_txt = os.path.join(newroot, 'typeids.txt')
         print 'loading', typeids_txt
-        typeids = {}
+        with open(typeids_txt) as f:
+            typeids = TypeIdsMap(f.readlines(), self.gdb)
+        return typeids
+
+
+class TypeIdsMap(object):
+    def __init__(self, lines, gdb):
+        self.lines = lines
+        self.gdb = gdb
+        self.line2offset = {0: 0}
+        self.offset2descr = {0: "(null typeid)"}
+
+    def __getitem__(self, key):
+        value = self.get(key)
+        if value is None:
+            raise KeyError(key)
+        return value
+
+    def __contains__(self, key):
+        return self.get(key) is not None
+
+    def _fetchline(self, linenum):
+        if linenum in self.line2offset:
+            return self.line2offset[linenum]
+        line = self.lines[linenum]
+        member, descr = map(str.strip, line.split(None, 1))
         if sys.maxint < 2**32:
             TIDT = "int*"
         else:
             TIDT = "char*"
-        with open(typeids_txt) as f:
-            for line in f:
-                member, descr = map(str.strip, line.split(None, 1))
-                expr = ("((%s)(&pypy_g_typeinfo.%s)) - (%s)&pypy_g_typeinfo"
-                           % (TIDT, member, TIDT))
-                offset = int(self.gdb.parse_and_eval(expr))
-                typeids[offset] = descr
-        return typeids
+        expr = ("((%s)(&pypy_g_typeinfo.%s)) - (%s)&pypy_g_typeinfo"
+                   % (TIDT, member, TIDT))
+        offset = int(self.gdb.parse_and_eval(expr))
+        self.line2offset[linenum] = offset
+        self.offset2descr[offset] = descr
+        return offset
+
+    def get(self, offset, default=None):
+        # binary search through the lines, asking gdb to parse stuff lazily
+        if offset in self.offset2descr:
+            return self.offset2descr[offset]
+        if not (0 < offset < sys.maxint):
+            return None
+        linerange = (0, len(self.lines))
+        while linerange[0] < linerange[1]:
+            linemiddle = (linerange[0] + linerange[1]) >> 1
+            offsetmiddle = self._fetchline(linemiddle)
+            if offsetmiddle == offset:
+                return self.offset2descr[offset]
+            elif offsetmiddle < offset:
+                linerange = (linemiddle + 1, linerange[1])
+            else:
+                linerange = (linerange[0], linemiddle)
+        return None
 
 
 def is_ptr(type, gdb):
diff --git a/pypy/tool/test/test_gdb_pypy.py b/pypy/tool/test/test_gdb_pypy.py
--- a/pypy/tool/test/test_gdb_pypy.py
+++ b/pypy/tool/test/test_gdb_pypy.py
@@ -1,11 +1,6 @@
 import py, sys
 from pypy.tool import gdb_pypy
 
-if sys.maxint < 2**32:
-    TIDT = "int*"
-else:
-    TIDT = "char*"
-
 class FakeGdb(object):
 
     COMMAND_NONE = -1
@@ -17,8 +12,10 @@
     def __init__(self, exprs, progspace=None):
         self.exprs = exprs
         self.progspace = progspace
+        self._parsed = []
 
     def parse_and_eval(self, expr):
+        self._parsed.append(expr)
         return self.exprs[expr]
 
     def current_progspace(self):
@@ -105,29 +102,39 @@
     hdr = gdb_pypy.lookup(obj, 'gcheader')
     assert hdr['h_tid'] == 123
 
+def exprmember(n):
+    if sys.maxint < 2**32:
+        TIDT = "int*"
+    else:
+        TIDT = "char*"
+    return ('((%s)(&pypy_g_typeinfo.member%d)) - (%s)&pypy_g_typeinfo'
+            % (TIDT, n, TIDT))
+
 def test_load_typeids(tmpdir):
     exe = tmpdir.join('testing_1').join('pypy-c')
     typeids = tmpdir.join('typeids.txt')
     typeids.write("""
-member0    GcStruct xxx {}
+member0    ?
+member1    GcStruct xxx {}
 """.strip())
     progspace = Mock(filename=str(exe))
-    exprs = {
-        '((%s)(&pypy_g_typeinfo.member0)) - (%s)&pypy_g_typeinfo'
-            % (TIDT, TIDT): 0,
-        }
+    exprs = {exprmember(1): 111}
     gdb = FakeGdb(exprs, progspace)
     cmd = gdb_pypy.RPyType(gdb)
     typeids = cmd.load_typeids(progspace)
-    assert typeids[0] == 'GcStruct xxx {}'
+    assert typeids[0] == '(null typeid)'
+    assert typeids[111] == 'GcStruct xxx {}'
+    py.test.raises(KeyError, "typeids[50]")
+    py.test.raises(KeyError, "typeids[150]")
 
 def test_RPyType(tmpdir):
     exe = tmpdir.join('pypy-c')
     typeids = tmpdir.join('typeids.txt')
     typeids.write("""
-member0    GcStruct xxx {}
-member1    GcStruct yyy {}
-member2    GcStruct zzz {}
+member0    ?
+member1    GcStruct xxx {}
+member2    GcStruct yyy {}
+member3    GcStruct zzz {}
 """.strip())
     #
     progspace = Mock(filename=str(exe))
@@ -141,12 +148,9 @@
     myvar = Value(d)
     exprs = {
         '*myvar': myvar,
-        '((%s)(&pypy_g_typeinfo.member0)) - (%s)&pypy_g_typeinfo'
-            % (TIDT, TIDT): 0,
-        '((%s)(&pypy_g_typeinfo.member1)) - (%s)&pypy_g_typeinfo'
-            % (TIDT, TIDT): 123,
-        '((%s)(&pypy_g_typeinfo.member2)) - (%s)&pypy_g_typeinfo'
-            % (TIDT, TIDT): 456,
+        exprmember(1): 0,
+        exprmember(2): 123,
+        exprmember(3): 456,
         }
     gdb = FakeGdb(exprs, progspace)
     cmd = gdb_pypy.RPyType(gdb)
@@ -192,3 +196,23 @@
 
     mylist.type.target().tag = None
     assert gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb) is None
+
+def test_typeidsmap():
+    gdb = FakeGdb({exprmember(1): 111,
+                   exprmember(2): 222,
+                   exprmember(3): 333})
+    typeids = gdb_pypy.TypeIdsMap(["member0  ?\n",
+                                   "member1  FooBar\n",
+                                   "member2  Baz\n",
+                                   "member3  Bok\n"], gdb)
+    assert gdb._parsed == []
+    assert typeids.get(111) == "FooBar"
+    assert gdb._parsed == [exprmember(2), exprmember(1)]
+    assert typeids.get(222) == "Baz"
+    assert gdb._parsed == [exprmember(2), exprmember(1)]
+    assert typeids.get(333) == "Bok"
+    assert gdb._parsed == [exprmember(2), exprmember(1), exprmember(3)]
+    assert typeids.get(400) == None
+    assert typeids.get(300) == None
+    assert typeids.get(200) == None
+    assert typeids.get(100) == None

From noreply at buildbot.pypy.org  Wed Oct 16 13:27:32 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 13:27:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: Meh
Message-ID: <20131016112732.D3B231C0203@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67412:17bccf6fbfbb
Date: 2013-10-16 13:26 +0200
http://bitbucket.org/pypy/pypy/changeset/17bccf6fbfbb/

Log:	Meh

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -208,11 +208,16 @@
         items = chars['items']
         res = []
         for i in range(min(length, MAX_DISPLAY_LENGTH)):
+            c = items[i]
             try:
-                res.append(chr(items[i]))
+                res.append(chr(c))
             except ValueError:
                 # it's a gdb.Value so it has "121 'y'" as repr
-                res.append(chr(int(str(items[0]).split(" ")[0])))
+                try:
+                    res.append(chr(int(str(c).split(" ")[0])))
+                except ValueError:
+                    # meh?
+                    res.append(repr(c))
         if length > MAX_DISPLAY_LENGTH:
             res.append('...')
         string = ''.join(res)

From noreply at buildbot.pypy.org  Wed Oct 16 13:52:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 13:52:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fixes
Message-ID: <20131016115226.06BD11C00ED@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67413:82101cce8799
Date: 2013-10-16 11:51 +0000
http://bitbucket.org/pypy/pypy/changeset/82101cce8799/

Log:	Fixes

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -89,7 +89,10 @@
             obj = self.gdb.parse_and_eval(arg)
             hdr = lookup(obj, '_gcheader')
             tid = hdr['h_tid']
-            offset = tid & 0xFFFFFFFF # 64bit only
+            if sys.maxint < 2**32:
+                offset = tid & 0xFFFF     # 32bit
+            else:
+                offset = tid & 0xFFFFFFFF # 64bit
             offset = int(offset) # convert from gdb.Value to python int
 
         typeids = self.get_typeids()
@@ -99,7 +102,10 @@
             return 'Cannot find the type with offset %d' % offset
 
     def get_typeids(self):
-        progspace = self.gdb.current_progspace()
+        try:
+            progspace = self.gdb.current_progspace()
+        except AttributeError:
+            progspace = None
         try:
             return self.prog2typeids[progspace]
         except KeyError:
@@ -111,7 +117,7 @@
         """
         Returns a mapping offset --> description
         """
-        exename = progspace.filename
+        exename = getattr(progspace, 'filename', '')
         root = os.path.dirname(exename)
         # XXX The same information is found in
         # XXX   pypy_g_rpython_memory_gctypelayout_GCData.gcd_inst_typeids_z
@@ -156,6 +162,7 @@
         offset = int(self.gdb.parse_and_eval(expr))
         self.line2offset[linenum] = offset
         self.offset2descr[offset] = descr
+        #print '%r -> %r -> %r' % (linenum, offset, descr)
         return offset
 
     def get(self, offset, default=None):

From noreply at buildbot.pypy.org  Wed Oct 16 14:09:30 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 14:09:30 +0200 (CEST)
Subject: [pypy-commit] pypy refactor-str-types: merge default
Message-ID: <20131016120930.ED58E1C0203@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: refactor-str-types
Changeset: r67414:4a07c4749c48
Date: 2013-10-15 11:55 +0200
http://bitbucket.org/pypy/pypy/changeset/4a07c4749c48/

Log:	merge default

diff too long, truncating to 2000 out of 22775 lines

diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -1780,7 +1780,19 @@
             # error if this argument is not allowed with other previously
             # seen arguments, assuming that actions that use the default
             # value don't really count as "present"
-            if argument_values is not action.default:
+
+            # XXX PyPy bug-to-bug compatibility: "is" on primitive types
+            # is not consistent in CPython.  We'll assume it is close
+            # enough for ints (which is true only for "small ints"), but
+            # for floats and longs and complexes we'll go for the option
+            # of forcing "is" to say False, like it usually does on
+            # CPython.  A fix is pending on CPython trunk
+            # (http://bugs.python.org/issue18943) but that might change
+            # the details of the semantics and so not be applied to 2.7.
+            # See the line AA below.
+
+            if (argument_values is not action.default or
+                    type(argument_values) in (float, long, complex)):  # AA
                 seen_non_default_actions.add(action)
                 for conflict_action in action_conflicts.get(action, []):
                     if conflict_action in seen_non_default_actions:
diff --git a/lib-python/2.7/test/keycert.pem b/lib-python/2.7/test/keycert.pem
--- a/lib-python/2.7/test/keycert.pem
+++ b/lib-python/2.7/test/keycert.pem
@@ -1,32 +1,31 @@
------BEGIN RSA PRIVATE KEY-----
-MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L
-opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH
-fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB
-AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU
-D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA
-IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM
-oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0
-ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/
-loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j
-oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA
-z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq
-ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV
-q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU=
------END RSA PRIVATE KEY-----
+-----BEGIN PRIVATE KEY-----
+MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm
+LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0
+ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP
+USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt
+CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq
+SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK
+UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y
+BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ
+ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5
+oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik
+eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F
+0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS
+x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/
+SPIXQuT8RMPDVNQ=
+-----END PRIVATE KEY-----
 -----BEGIN CERTIFICATE-----
-MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD
-VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x
-IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT
-U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1
-NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl
-bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m
-dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj
-aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh
-m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8
-M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn
-fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC
-AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb
-08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx
-CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/
-iHkC6gGdBJhogs4=
+MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw
+MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH
+Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k
+YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
+gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7
+6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt
+pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw
+FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd
+BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G
+lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1
+CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX
 -----END CERTIFICATE-----
diff --git a/lib-python/2.7/test/sha256.pem b/lib-python/2.7/test/sha256.pem
--- a/lib-python/2.7/test/sha256.pem
+++ b/lib-python/2.7/test/sha256.pem
@@ -1,129 +1,128 @@
 # Certificate chain for https://sha256.tbs-internet.com
- 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=sha-256 production/CN=sha256.tbs-internet.com
-   i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC
+ 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=Certificats TBS X509/CN=ecom.tbs-x509.com
+   i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business
 -----BEGIN CERTIFICATE-----
-MIIGXTCCBUWgAwIBAgIRAMmag+ygSAdxZsbyzYjhuW0wDQYJKoZIhvcNAQELBQAw
-gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl
+MIIGTjCCBTagAwIBAgIQOh3d9dNDPq1cSdJmEiMpqDANBgkqhkiG9w0BAQUFADCB
+yTELMAkGA1UEBhMCRlIxETAPBgNVBAgTCENhbHZhZG9zMQ0wCwYDVQQHEwRDYWVu
+MRUwEwYDVQQKEwxUQlMgSU5URVJORVQxSDBGBgNVBAsTP1Rlcm1zIGFuZCBDb25k
+aXRpb25zOiBodHRwOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0EvcmVwb3NpdG9y
+eTEYMBYGA1UECxMPVEJTIElOVEVSTkVUIENBMR0wGwYDVQQDExRUQlMgWDUwOSBD
+QSBidXNpbmVzczAeFw0xMTAxMjUwMDAwMDBaFw0xMzAyMDUyMzU5NTlaMIHHMQsw
+CQYDVQQGEwJGUjEOMAwGA1UEERMFMTQwMDAxETAPBgNVBAgTCENhbHZhZG9zMQ0w
+CwYDVQQHEwRDQUVOMRswGQYDVQQJExIyMiBydWUgZGUgQnJldGFnbmUxFTATBgNV
+BAoTDFRCUyBJTlRFUk5FVDEXMBUGA1UECxMOMDAwMiA0NDA0NDM4MTAxHTAbBgNV
+BAsTFENlcnRpZmljYXRzIFRCUyBYNTA5MRowGAYDVQQDExFlY29tLnRicy14NTA5
+LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKRrlHUnJ++1lpcg
+jtYco7cdmRe+EEfTmwPfCdfV3G1QfsTSvY6FfMpm/83pqHfT+4ANwr18wD9ZrAEN
+G16mf9VdCGK12+TP7DmqeZyGIqlFFoahQnmb8EarvE43/1UeQ2CV9XmzwZvpqeli
+LfXsFonawrY3H6ZnMwS64St61Z+9gdyuZ/RbsoZBbT5KUjDEG844QRU4OT1IGeEI
+eY5NM5RNIh6ZNhVtqeeCxMS7afONkHQrOco73RdSTRck/Hj96Ofl3MHNHryr+AMK
+DGFk1kLCZGpPdXtkxXvaDeQoiYDlil26CWc+YK6xyDPMdsWvoG14ZLyCpzMXA7/7
+4YAQRH0CAwEAAaOCAjAwggIsMB8GA1UdIwQYMBaAFBoJBMz5CY+7HqDO1KQUf0vV
+I1jNMB0GA1UdDgQWBBQgOU8HsWzbmD4WZP5Wtdw7jca2WDAOBgNVHQ8BAf8EBAMC
+BaAwDAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw
+TAYDVR0gBEUwQzBBBgsrBgEEAYDlNwIBATAyMDAGCCsGAQUFBwIBFiRodHRwczov
+L3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL0NQUzEwdwYDVR0fBHAwbjA3oDWgM4Yx
+aHR0cDovL2NybC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNy
+bDAzoDGgL4YtaHR0cDovL2NybC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5l
+c3MuY3JsMIGwBggrBgEFBQcBAQSBozCBoDA9BggrBgEFBQcwAoYxaHR0cDovL2Ny
+dC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNydDA5BggrBgEF
+BQcwAoYtaHR0cDovL2NydC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5lc3Mu
+Y3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wMwYDVR0R
+BCwwKoIRZWNvbS50YnMteDUwOS5jb22CFXd3dy5lY29tLnRicy14NTA5LmNvbTAN
+BgkqhkiG9w0BAQUFAAOCAQEArT4NHfbY87bGAw8lPV4DmHlmuDuVp/y7ltO3Ynse
+3Rz8RxW2AzuO0Oy2F0Cu4yWKtMyEyMXyHqWtae7ElRbdTu5w5GwVBLJHClCzC8S9
+SpgMMQTx3Rgn8vjkHuU9VZQlulZyiPK7yunjc7c310S9FRZ7XxOwf8Nnx4WnB+No
+WrfApzhhQl31w+RyrNxZe58hCfDDHmevRvwLjQ785ZoQXJDj2j3qAD4aI2yB8lB5
+oaE1jlCJzC7Kmz/Y9jzfmv/zAs1LQTm9ktevv4BTUFaGjv9jxnQ1xnS862ZiouLW
+zZYIlYPf4F6JjXGiIQgQRglILUfq3ftJd9/ok9W9ZF8h8w==
+-----END CERTIFICATE-----
+ 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business
+   i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root
+-----BEGIN CERTIFICATE-----
+MIIFPzCCBCegAwIBAgIQDlBz/++iRSmLDeVRHT/hADANBgkqhkiG9w0BAQUFADBv
+MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk
+ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF
+eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDcwOTE4MTkyMlow
+gckxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl
 bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u
 ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv
-cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg
-Q0EgU0dDMB4XDTEwMDIxODAwMDAwMFoXDTEyMDIxOTIzNTk1OVowgcsxCzAJBgNV
-BAYTAkZSMQ4wDAYDVQQREwUxNDAwMDERMA8GA1UECBMIQ2FsdmFkb3MxDTALBgNV
-BAcTBENBRU4xGzAZBgNVBAkTEjIyIHJ1ZSBkZSBCcmV0YWduZTEVMBMGA1UEChMM
-VEJTIElOVEVSTkVUMRcwFQYDVQQLEw4wMDAyIDQ0MDQ0MzgxMDEbMBkGA1UECxMS
-c2hhLTI1NiBwcm9kdWN0aW9uMSAwHgYDVQQDExdzaGEyNTYudGJzLWludGVybmV0
-LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKbuM8VT7f0nntwu
-N3F7v9KIBlhKNAxqCrziOXU5iqUt8HrQB3DtHbdmII+CpVUlwlmepsx6G+srEZ9a
-MIGAy0nxi5aLb7watkyIdPjJTMvTUBQ/+RPWzt5JtYbbY9BlJ+yci0dctP74f4NU
-ISLtlrEjUbf2gTohLrcE01TfmOF6PDEbB5PKDi38cB3NzKfizWfrOaJW6Q1C1qOJ
-y4/4jkUREX1UFUIxzx7v62VfjXSGlcjGpBX1fvtABQOSLeE0a6gciDZs1REqroFf
-5eXtqYphpTa14Z83ITXMfgg5Nze1VtMnzI9Qx4blYBw4dgQVEuIsYr7FDBOITDzc
-VEVXZx0CAwEAAaOCAj8wggI7MB8GA1UdIwQYMBaAFAdEdoWTKLx/bXjSCuv6TEvf
-2YIfMB0GA1UdDgQWBBSJKI/AYVI9RQNY0QPIqc8ej2QivTAOBgNVHQ8BAf8EBAMC
-BaAwDAYDVR0TAQH/BAIwADA0BgNVHSUELTArBggrBgEFBQcDAQYIKwYBBQUHAwIG
-CisGAQQBgjcKAwMGCWCGSAGG+EIEATBMBgNVHSAERTBDMEEGCysGAQQBgOU3AgQB
-MDIwMAYIKwYBBQUHAgEWJGh0dHBzOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0Ev
-Q1BTNDBtBgNVHR8EZjBkMDKgMKAuhixodHRwOi8vY3JsLnRicy1pbnRlcm5ldC5j
-b20vVEJTWDUwOUNBU0dDLmNybDAuoCygKoYoaHR0cDovL2NybC50YnMteDUwOS5j
-b20vVEJTWDUwOUNBU0dDLmNybDCBpgYIKwYBBQUHAQEEgZkwgZYwOAYIKwYBBQUH
-MAKGLGh0dHA6Ly9jcnQudGJzLWludGVybmV0LmNvbS9UQlNYNTA5Q0FTR0MuY3J0
-MDQGCCsGAQUFBzAChihodHRwOi8vY3J0LnRicy14NTA5LmNvbS9UQlNYNTA5Q0FT
-R0MuY3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wPwYD
-VR0RBDgwNoIXc2hhMjU2LnRicy1pbnRlcm5ldC5jb22CG3d3dy5zaGEyNTYudGJz
-LWludGVybmV0LmNvbTANBgkqhkiG9w0BAQsFAAOCAQEAA5NL0D4QSqhErhlkdPmz
-XtiMvdGL+ZehM4coTRIpasM/Agt36Rc0NzCvnQwKE+wkngg1Gy2qe7Q0E/ziqBtB
-fZYzdVgu1zdiL4kTaf+wFKYAFGsFbyeEmXysy+CMwaNoF2vpSjCU1UD56bEnTX/W
-fxVZYxtBQUpnu2wOsm8cDZuZRv9XrYgAhGj9Tt6F0aVHSDGn59uwShG1+BVF/uju
-SCyPTTjL1oc7YElJUzR/x4mQJYvtQI8gDIDAGEOs7v3R/gKa5EMfbUQUI4C84UbI
-Yz09Jdnws/MkC/Hm1BZEqk89u7Hvfv+oHqEb0XaUo0TDfsxE0M1sMdnLb91QNQBm
-UQ==
------END CERTIFICATE-----
- 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC
-   i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root
------BEGIN CERTIFICATE-----
-MIIFVjCCBD6gAwIBAgIQXpDZ0ETJMV02WTx3GTnhhTANBgkqhkiG9w0BAQUFADBv
-MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk
-ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF
-eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDYyNDE5MDYzMFow
-gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl
-bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u
-ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv
-cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg
-Q0EgU0dDMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgOkO3f7wzN6
-rOjg45tR5vjBfzK7qmV9IBxb/QW9EEXxG+E7FNhZqQLtwGBKoSsHTnQqV75wWMk0
-9tinWvftBkSpj5sTi/8cbzJfUvTSVYh3Qxv6AVVjMMH/ruLjE6y+4PoaPs8WoYAQ
-ts5R4Z1g8c/WnTepLst2x0/Wv7GmuoQi+gXvHU6YrBiu7XkeYhzc95QdviWSJRDk
-owhb5K43qhcvjRmBfO/paGlCliDGZp8mHwrI21mwobWpVjTxZRwYO3bd4+TGcI4G
-Ie5wmHwE8F7SK1tgSqbBacKjDa93j7txKkfz/Yd2n7TGqOXiHPsJpG655vrKtnXk
-9vs1zoDeJQIDAQABo4IBljCCAZIwHQYDVR0OBBYEFAdEdoWTKLx/bXjSCuv6TEvf
-2YIfMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMCAGA1UdJQQZ
-MBcGCisGAQQBgjcKAwMGCWCGSAGG+EIEATAYBgNVHSAEETAPMA0GCysGAQQBgOU3
-AgQBMHsGA1UdHwR0MHIwOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0Fk
-ZFRydXN0RXh0ZXJuYWxDQVJvb3QuY3JsMDagNKAyhjBodHRwOi8vY3JsLmNvbW9k
-by5uZXQvQWRkVHJ1c3RFeHRlcm5hbENBUm9vdC5jcmwwgYAGCCsGAQUFBwEBBHQw
-cjA4BggrBgEFBQcwAoYsaHR0cDovL2NydC5jb21vZG9jYS5jb20vQWRkVHJ1c3RV
-VE5TR0NDQS5jcnQwNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQuY29tb2RvLm5ldC9B
-ZGRUcnVzdFVUTlNHQ0NBLmNydDARBglghkgBhvhCAQEEBAMCAgQwDQYJKoZIhvcN
-AQEFBQADggEBAK2zEzs+jcIrVK9oDkdDZNvhuBYTdCfpxfFs+OAujW0bIfJAy232
-euVsnJm6u/+OrqKudD2tad2BbejLLXhMZViaCmK7D9nrXHx4te5EP8rL19SUVqLY
-1pTnv5dhNgEgvA7n5lIzDSYs7yRLsr7HJsYPr6SeYSuZizyX1SNz7ooJ32/F3X98
-RB0Mlc/E0OyOrkQ9/y5IrnpnaSora8CnUrV5XNOg+kyCz9edCyx4D5wXYcwZPVWz
-8aDqquESrezPyjtfi4WRO4s/VD3HLZvOxzMrWAVYCDG9FxaOhF0QGuuG1F7F3GKV
-v6prNyCl016kRl2j1UT+a7gLd8fA25A4C9E=
+cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEdMBsGA1UEAxMUVEJTIFg1MDkg
+Q0EgYnVzaW5lc3MwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDB1PAU
+qudCcz3tmyGcf+u6EkZqonKKHrV4gZYbvVkIRojmmlhfi/jwvpHvo8bqSt/9Rj5S
+jhCDW0pcbI+IPPtD1Jy+CHNSfnMqVDy6CKQ3p5maTzCMG6ZT+XjnvcND5v+FtaiB
+xk1iCX6uvt0jeUtdZvYbyytsSDE6c3Y5//wRxOF8tM1JxibwO3pyER26jbbN2gQz
+m/EkdGjLdJ4svPk23WDAvQ6G0/z2LcAaJB+XLfqRwfQpHQvfKa1uTi8PivC8qtip
+rmNQMMPMjxSK2azX8cKjjTDJiUKaCb4VHlJDWKEsCFRpgJAoAuX8f7Yfs1M4esGo
+sWb3PGspK3O22uIlAgMBAAGjggF6MIIBdjAdBgNVHQ4EFgQUGgkEzPkJj7seoM7U
+pBR/S9UjWM0wDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwGAYD
+VR0gBBEwDzANBgsrBgEEAYDlNwIBATB7BgNVHR8EdDByMDigNqA0hjJodHRwOi8v
+Y3JsLmNvbW9kb2NhLmNvbS9BZGRUcnVzdEV4dGVybmFsQ0FSb290LmNybDA2oDSg
+MoYwaHR0cDovL2NybC5jb21vZG8ubmV0L0FkZFRydXN0RXh0ZXJuYWxDQVJvb3Qu
+Y3JsMIGGBggrBgEFBQcBAQR6MHgwOwYIKwYBBQUHMAKGL2h0dHA6Ly9jcnQuY29t
+b2RvY2EuY29tL0FkZFRydXN0VVROU2VydmVyQ0EuY3J0MDkGCCsGAQUFBzAChi1o
+dHRwOi8vY3J0LmNvbW9kby5uZXQvQWRkVHJ1c3RVVE5TZXJ2ZXJDQS5jcnQwEQYJ
+YIZIAYb4QgEBBAQDAgIEMA0GCSqGSIb3DQEBBQUAA4IBAQA7mqrMgk/MrE6QnbNA
+h4nRCn2ti4bg4w2C3lB6bSvRPnYwuNw9Jb8vuKkNFzRDxNJXqVDZdfFW5CVQJuyd
+nfAx83+wk+spzvFaE1KhFYfN9G9pQfXUfvDRoIcJgPEKUXL1wRiOG+IjU3VVI8pg
+IgqHkr7ylln5i5zCiFAPuIJmYUSFg/gxH5xkCNcjJqqrHrHatJr6Qrrke93joupw
+oU1njfAcZtYp6fbiK6u2b1pJqwkVBE8RsfLnPhRj+SFbpvjv8Od7o/ieJhFIYQNU
+k2jX2u8qZnAiNw93LZW9lpYjtuvMXq8QQppENNja5b53q7UwI+lU7ZGjZ7quuESp
+J6/5
 -----END CERTIFICATE-----
  2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root
-   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC
+   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware
 -----BEGIN CERTIFICATE-----
-MIIEZjCCA06gAwIBAgIQUSYKkxzif5zDpV954HKugjANBgkqhkiG9w0BAQUFADCB
-kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
+MIIETzCCAzegAwIBAgIQHM5EYpUZep1jUvnyI6m2mDANBgkqhkiG9w0BAQUFADCB
+lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
 Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho
-dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw
-IFNHQzAeFw0wNTA2MDcwODA5MTBaFw0xOTA2MjQxOTA2MzBaMG8xCzAJBgNVBAYT
-AlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0
-ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB
-IFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC39xoz5vIABC05
-4E5b7R+8bA/Ntfojts7emxEzl6QpTH2Tn71KvJPtAxrjj8/lbVBa1pcplFqAsEl6
-2y6V/bjKvzc4LR4+kUGtcFbH8E8/6DKedMrIkFTpxl8PeJ2aQDwOrGGqXhSPnoeh
-alDc15pOrwWzpnGUnHGzUGAKxxOdOAeGAqjpqGkmGJCrTLBPI6s6T4TY386f4Wlv
-u9dC12tE5Met7m1BX3JacQg3s3llpFmglDf3AC8NwpJy2tA4ctsUqEXEXSp9t7TW
-xO6szRNEt8kr3UMAJfphuWlqWCMRt6czj1Z1WfXNKddGtworZbbTQm8Vsrh7++/p
-XVPVNFonAgMBAAGjgdgwgdUwHwYDVR0jBBgwFoAUUzLRs89/+uDxoF2FTpLSnkUd
-tE8wHQYDVR0OBBYEFK29mHo0tCb3+sQmVO8DveAky1QaMA4GA1UdDwEB/wQEAwIB
-BjAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBAjAgBgNVHSUEGTAX
-BgorBgEEAYI3CgMDBglghkgBhvhCBAEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDov
-L2NybC51c2VydHJ1c3QuY29tL1VUTi1EQVRBQ29ycFNHQy5jcmwwDQYJKoZIhvcN
-AQEFBQADggEBAMbuUxdoFLJRIh6QWA2U/b3xcOWGLcM2MY9USEbnLQg3vGwKYOEO
-rVE04BKT6b64q7gmtOmWPSiPrmQH/uAB7MXjkesYoPF1ftsK5p+R26+udd8jkWjd
-FwBaS/9kbHDrARrQkNnHptZt9hPk/7XJ0h4qy7ElQyZ42TCbTg0evmnv3+r+LbPM
-+bDdtRTKkdSytaX7ARmjR3mfnYyVhzT4HziS2jamEfpr62vp3EV4FTkG101B5CHI
-3C+H0be/SGB1pWLLJN47YaApIKa+xWycxOkKaSLvkTr6Jq/RW0GnOuL4OAdCq8Fb
-+M5tug8EPzI0rNwEKNdwMBQmBsTkm5jVz3g=
+dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt
+SGFyZHdhcmUwHhcNMDUwNjA3MDgwOTEwWhcNMTkwNzA5MTgxOTIyWjBvMQswCQYD
+VQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0
+IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5h
+bCBDQSBSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/caM+by
+AAQtOeBOW+0fvGwPzbX6I7bO3psRM5ekKUx9k5+9SryT7QMa44/P5W1QWtaXKZRa
+gLBJetsulf24yr83OC0ePpFBrXBWx/BPP+gynnTKyJBU6cZfD3idmkA8Dqxhql4U
+j56HoWpQ3NeaTq8Fs6ZxlJxxs1BgCscTnTgHhgKo6ahpJhiQq0ywTyOrOk+E2N/O
+n+Fpb7vXQtdrROTHre5tQV9yWnEIN7N5ZaRZoJQ39wAvDcKSctrQOHLbFKhFxF0q
+fbe01sTurM0TRLfJK91DACX6YblpalgjEbenM49WdVn1zSnXRrcKK2W200JvFbK4
+e/vv6V1T1TRaJwIDAQABo4G9MIG6MB8GA1UdIwQYMBaAFKFyXyYbKJhDlV0HN9WF
+lp1L0sNFMB0GA1UdDgQWBBStvZh6NLQm9/rEJlTvA73gJMtUGjAOBgNVHQ8BAf8E
+BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhCAQEEBAMCAQIwRAYDVR0f
+BD0wOzA5oDegNYYzaHR0cDovL2NybC51c2VydHJ1c3QuY29tL1VUTi1VU0VSRmly
+c3QtSGFyZHdhcmUuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQByQhANOs4kClrwF8BW
+onvUOGCSjRK52zYZgDXYNjDtmr5rJ6NyPFDNn+JxkLpjYetIFMTbSRe679Bt8m7a
+gIAoQYFQtxMuyLnJegB2aEbQiIxh/tC21UcFF7ktdnDoTlA6w3pLuvunaI84Of3o
+2YBrhzkTbCfaYk5JRlTpudW9DkUkHBsyx3nknPKnplkIGaK0jgn8E0n+SFabYaHk
+I9LroYT/+JtLefh9lgBdAgVv0UPbzoGfuDsrk/Zh+UrgbLFpHoVnElhzbkh64Z0X
+OGaJunQc68cCZu5HTn/aK7fBGMcVflRCXLVEQpU9PIAdGA8Ynvg684t8GMaKsRl1
+jIGZ
 -----END CERTIFICATE-----
- 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC
-   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC
+ 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware
+   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware
 -----BEGIN CERTIFICATE-----
-MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB
-kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
+MIIEdDCCA1ygAwIBAgIQRL4Mi1AAJLQR0zYq/mUK/TANBgkqhkiG9w0BAQUFADCB
+lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
 Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho
-dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw
-IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG
-EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD
-VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu
-dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN
-BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6
-E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ
-D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK
-4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq
-lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW
-bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB
-o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT
-MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js
-LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr
-BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB
-AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft
-Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj
-j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH
-KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv
-2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3
-mfnGV/TJVTl4uix5yaaIK/QI
+dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt
+SGFyZHdhcmUwHhcNOTkwNzA5MTgxMDQyWhcNMTkwNzA5MTgxOTIyWjCBlzELMAkG
+A1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0eTEe
+MBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8v
+d3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdh
+cmUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCx98M4P7Sof885glFn
+0G2f0v9Y8+efK+wNiVSZuTiZFvfgIXlIwrthdBKWHTxqctU8EGc6Oe0rE81m65UJ
+M6Rsl7HoxuzBdXmcRl6Nq9Bq/bkqVRcQVLMZ8Jr28bFdtqdt++BxF2uiiPsA3/4a
+MXcMmgF6sTLjKwEHOG7DpV4jvEWbe1DByTCP2+UretNb+zNAHqDVmBe8i4fDidNd
+oI6yqqr2jmmIBsX6iSHzCJ1pLgkzmykNRg+MzEk0sGlRvfkGzWitZky8PqxhvQqI
+DsjfPe58BEydCl5rkdbux+0ojatNh4lz0G6k0B4WixThdkQDf2Os5M1JnMWS9Ksy
+oUhbAgMBAAGjgbkwgbYwCwYDVR0PBAQDAgHGMA8GA1UdEwEB/wQFMAMBAf8wHQYD
+VR0OBBYEFKFyXyYbKJhDlV0HN9WFlp1L0sNFMEQGA1UdHwQ9MDswOaA3oDWGM2h0
+dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VVE4tVVNFUkZpcnN0LUhhcmR3YXJlLmNy
+bDAxBgNVHSUEKjAoBggrBgEFBQcDAQYIKwYBBQUHAwUGCCsGAQUFBwMGBggrBgEF
+BQcDBzANBgkqhkiG9w0BAQUFAAOCAQEARxkP3nTGmZev/K0oXnWO6y1n7k57K9cM
+//bey1WiCuFMVGWTYGufEpytXoMs61quwOQt9ABjHbjAbPLPSbtNk28Gpgoiskli
+CE7/yMgUsogWXecB5BKV5UU0s4tpvc+0hY91UZ59Ojg6FEgSxvunOxqNDYJAB+gE
+CJChicsZUN/KHAG8HQQZexB2lzvukJDKxA4fFm517zP4029bHpbj4HR3dHuKom4t
+3XbWOTCC8KucUvIqx69JXn7HaOWCgchqJ/kniCrVWFCVH/A7HFe7fRQ5YiuayZSS
+KqMiDP+JJn1fIytH1xUdqWqeUQ0qUZ6B+dQ7XnASfxAynB67nfhmqA==
 -----END CERTIFICATE-----
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -111,13 +111,12 @@
         if test_support.verbose:
             sys.stdout.write("\n" + pprint.pformat(p) + "\n")
         self.assertEqual(p['subject'],
-                         ((('countryName', u'US'),),
-                          (('stateOrProvinceName', u'Delaware'),),
-                          (('localityName', u'Wilmington'),),
-                          (('organizationName', u'Python Software Foundation'),),
-                          (('organizationalUnitName', u'SSL'),),
-                          (('commonName', u'somemachine.python.org'),)),
+                         ((('countryName', 'XY'),),
+                          (('localityName', 'Castle Anthrax'),),
+                          (('organizationName', 'Python Software Foundation'),),
+                          (('commonName', 'localhost'),))
                         )
+        self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),))
         # Issue #13034: the subjectAltName in some certificates
         # (notably projects.developer.nokia.com:443) wasn't parsed
         p = ssl._ssl._test_decode_cert(NOKIACERT)
diff --git a/lib-python/2.7/uuid.py b/lib-python/2.7/uuid.py
--- a/lib-python/2.7/uuid.py
+++ b/lib-python/2.7/uuid.py
@@ -127,30 +127,39 @@
         overriding the given 'hex', 'bytes', 'bytes_le', 'fields', or 'int'.
         """
 
-        if (
-            ((hex is not None) + (bytes is not None) + (bytes_le is not None) +
-             (fields is not None) + (int is not None)) != 1
-        ):
-            raise TypeError('need exactly one of hex, bytes, bytes_le, fields,'
-                            ' or int')
         if hex is not None:
+            if (bytes is not None or bytes_le is not None or
+                    fields is not None or int is not None):
+                raise TypeError('if the hex argument is given, bytes,'
+                                ' bytes_le, fields,  and int need to be None')
             hex = hex.replace('urn:', '').replace('uuid:', '')
             hex = hex.strip('{}').replace('-', '')
             if len(hex) != 32:
                 raise ValueError('badly formed hexadecimal UUID string')
             int = long(hex, 16)
-        if bytes_le is not None:
+        elif bytes_le is not None:
+            if bytes is not None or fields is not None or int is not None:
+                raise TypeError('if the bytes_le argument is given, bytes,'
+                                ' fields, and int need to be None')
             if len(bytes_le) != 16:
                 raise ValueError('bytes_le is not a 16-char string')
             bytes = (bytes_le[3] + bytes_le[2] + bytes_le[1] + bytes_le[0] +
                      bytes_le[5] + bytes_le[4] + bytes_le[7] + bytes_le[6] +
                      bytes_le[8:])
-        if bytes is not None:
+            int = (struct.unpack('>Q', bytes[:8])[0] << 64 |
+                   struct.unpack('>Q', bytes[8:])[0])
+        elif bytes is not None:
+            if fields is not None or int is not None:
+                raise TypeError('if the bytes argument is given, fields '
+                                'and int need to be None')
             if len(bytes) != 16:
                 raise ValueError('bytes is not a 16-char string')
             int = (struct.unpack('>Q', bytes[:8])[0] << 64 |
                    struct.unpack('>Q', bytes[8:])[0])
-        if fields is not None:
+        elif fields is not None:
+            if int is not None:
+                raise TypeError('if the fields argument is given, int needs'
+                                ' to be None')
             if len(fields) != 6:
                 raise ValueError('fields is not a 6-tuple')
             (time_low, time_mid, time_hi_version,
@@ -170,9 +179,12 @@
             clock_seq = (clock_seq_hi_variant << 8L) | clock_seq_low
             int = ((time_low << 96L) | (time_mid << 80L) |
                    (time_hi_version << 64L) | (clock_seq << 48L) | node)
-        if int is not None:
+        elif int is not None:
             if not 0 <= int < 1<<128L:
                 raise ValueError('int is out of range (need a 128-bit value)')
+        else:
+            raise TypeError('one of hex, bytes, bytes_le, fields,'
+                            ' or int need to be not None')
         if version is not None:
             if not 1 <= version <= 5:
                 raise ValueError('illegal version number')
@@ -182,7 +194,7 @@
             # Set the version number.
             int &= ~(0xf000 << 64L)
             int |= version << 76L
-        self.__dict__['int'] = int
+        object.__setattr__(self, 'int', int)
 
     def __cmp__(self, other):
         if isinstance(other, UUID):
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -1,6 +1,9 @@
 """Reimplementation of the standard extension module '_curses' using cffi."""
 
 import sys
+if sys.platform == 'win32':
+    #This module does not exist in windows
+    raise ImportError('No module named _curses')
 from functools import wraps
 
 from cffi import FFI
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -363,9 +363,11 @@
     pass
 
 
-def connect(database, **kwargs):
-    factory = kwargs.get("factory", Connection)
-    return factory(database, **kwargs)
+def connect(database, timeout=5.0, detect_types=0, isolation_level="",
+                 check_same_thread=True, factory=None, cached_statements=100):
+    factory = Connection if not factory else factory
+    return factory(database, timeout, detect_types, isolation_level,
+                    check_same_thread, factory, cached_statements)
 
 
 def _unicode_text_factory(x):
diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
--- a/lib_pypy/_tkinter/__init__.py
+++ b/lib_pypy/_tkinter/__init__.py
@@ -22,6 +22,7 @@
 READABLE = tklib.TCL_READABLE
 WRITABLE = tklib.TCL_WRITABLE
 EXCEPTION = tklib.TCL_EXCEPTION
+DONT_WAIT = tklib.TCL_DONT_WAIT
 
 def create(screenName=None, baseName=None, className=None,
            interactive=False, wantobjects=False, wantTk=True,
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -4,7 +4,23 @@
 from . import TclError
 from .tclobj import TclObject, FromObj, AsObj, TypeCache
 
+import contextlib
 import sys
+import threading
+import time
+
+
+class _DummyLock(object):
+    "A lock-like object that does not do anything"
+    def acquire(self):
+        pass
+    def release(self):
+        pass
+    def __enter__(self):
+        pass
+    def __exit__(self, *exc):
+        pass
+
 
 def varname_converter(input):
     if isinstance(input, TclObject):
@@ -37,17 +53,18 @@
     def PythonCmd(clientData, interp, argc, argv):
         self = tkffi.from_handle(clientData)
         assert self.app.interp == interp
-        try:
-            args = [tkffi.string(arg) for arg in argv[1:argc]]
-            result = self.func(*args)
-            obj = AsObj(result)
-            tklib.Tcl_SetObjResult(interp, obj)
-        except:
-            self.app.errorInCmd = True
-            self.app.exc_info = sys.exc_info()
-            return tklib.TCL_ERROR
-        else:
-            return tklib.TCL_OK
+        with self.app._tcl_lock_released():
+            try:
+                args = [tkffi.string(arg) for arg in argv[1:argc]]
+                result = self.func(*args)
+                obj = AsObj(result)
+                tklib.Tcl_SetObjResult(interp, obj)
+            except:
+                self.app.errorInCmd = True
+                self.app.exc_info = sys.exc_info()
+                return tklib.TCL_ERROR
+            else:
+                return tklib.TCL_OK
 
     @tkffi.callback("Tcl_CmdDeleteProc")
     def PythonCmdDelete(clientData):
@@ -58,6 +75,8 @@
 
 
 class TkApp(object):
+    _busywaitinterval = 0.02  # 20ms.
+
     def __new__(cls, screenName, baseName, className,
                 interactive, wantobjects, wantTk, sync, use):
         if not wantobjects:
@@ -73,6 +92,12 @@
         self.quitMainLoop = False
         self.errorInCmd = False
 
+        if not self.threaded:
+            # TCL is not thread-safe, calls needs to be serialized.
+            self._tcl_lock = threading.Lock()
+        else:
+            self._tcl_lock = _DummyLock()
+
         self._typeCache = TypeCache()
         self._commands = {}
 
@@ -133,6 +158,13 @@
         if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
             raise RuntimeError("Calling Tcl from different appartment")
 
+    @contextlib.contextmanager
+    def _tcl_lock_released(self):
+        "Context manager to temporarily release the tcl lock."
+        self._tcl_lock.release()
+        yield
+        self._tcl_lock.acquire()
+
     def loadtk(self):
         # We want to guard against calling Tk_Init() multiple times
         err = tklib.Tcl_Eval(self.interp, "info exists     tk_version")
@@ -159,22 +191,25 @@
         flags=tklib.TCL_LEAVE_ERR_MSG
         if global_only:
             flags |= tklib.TCL_GLOBAL_ONLY
-        res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
-        if not res:
-            self.raiseTclError()
-        assert self._wantobjects
-        return FromObj(self, res)
+        with self._tcl_lock:
+            res = tklib.Tcl_GetVar2Ex(self.interp, name1, name2, flags)
+            if not res:
+                self.raiseTclError()
+            assert self._wantobjects
+            return FromObj(self, res)
 
     def _setvar(self, name1, value, global_only=False):
         name1 = varname_converter(name1)
+        # XXX Acquire tcl lock???
         newval = AsObj(value)
         flags=tklib.TCL_LEAVE_ERR_MSG
         if global_only:
             flags |= tklib.TCL_GLOBAL_ONLY
-        res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
-                                  newval, flags)
-        if not res:
-            self.raiseTclError()
+        with self._tcl_lock:
+            res = tklib.Tcl_SetVar2Ex(self.interp, name1, tkffi.NULL,
+                                      newval, flags)
+            if not res:
+                self.raiseTclError()
 
     def _unsetvar(self, name1, name2=None, global_only=False):
         name1 = varname_converter(name1)
@@ -183,9 +218,10 @@
         flags=tklib.TCL_LEAVE_ERR_MSG
         if global_only:
             flags |= tklib.TCL_GLOBAL_ONLY
-        res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
-        if res == tklib.TCL_ERROR:
-            self.raiseTclError()
+        with self._tcl_lock:
+            res = tklib.Tcl_UnsetVar2(self.interp, name1, name2, flags)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
 
     def getvar(self, name1, name2=None):
         return self._var_invoke(self._getvar, name1, name2)
@@ -219,9 +255,10 @@
         if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
             raise NotImplementedError("Call from another thread")
 
-        res = tklib.Tcl_CreateCommand(
-            self.interp, cmdName, _CommandData.PythonCmd,
-            clientData, _CommandData.PythonCmdDelete)
+        with self._tcl_lock:
+            res = tklib.Tcl_CreateCommand(
+                self.interp, cmdName, _CommandData.PythonCmd,
+                clientData, _CommandData.PythonCmdDelete)
         if not res:
             raise TclError("can't create Tcl command")
 
@@ -229,7 +266,8 @@
         if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread():
             raise NotImplementedError("Call from another thread")
 
-        res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
+        with self._tcl_lock:
+            res = tklib.Tcl_DeleteCommand(self.interp, cmdName)
         if res == -1:
             raise TclError("can't delete Tcl command")
 
@@ -256,11 +294,12 @@
                 tklib.Tcl_IncrRefCount(obj)
                 objects[i] = obj
 
-            res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
-            if res == tklib.TCL_ERROR:
-                self.raiseTclError()
-            else:
-                result = self._callResult()
+            with self._tcl_lock:
+                res = tklib.Tcl_EvalObjv(self.interp, argc, objects, flags)
+                if res == tklib.TCL_ERROR:
+                    self.raiseTclError()
+                else:
+                    result = self._callResult()
         finally:
             for obj in objects:
                 if obj:
@@ -280,17 +319,19 @@
 
     def eval(self, script):
         self._check_tcl_appartment()
-        res = tklib.Tcl_Eval(self.interp, script)
-        if res == tklib.TCL_ERROR:
-            self.raiseTclError()
-        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+        with self._tcl_lock:
+            res = tklib.Tcl_Eval(self.interp, script)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
 
     def evalfile(self, filename):
         self._check_tcl_appartment()
-        res = tklib.Tcl_EvalFile(self.interp, filename)
-        if res == tklib.TCL_ERROR:
-            self.raiseTclError()
-        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+        with self._tcl_lock:
+            res = tklib.Tcl_EvalFile(self.interp, filename)
+            if res == tklib.TCL_ERROR:
+                self.raiseTclError()
+            return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
 
     def split(self, arg):
         if isinstance(arg, tuple):
@@ -375,7 +416,10 @@
             if self.threaded:
                 result = tklib.Tcl_DoOneEvent(0)
             else:
-                raise NotImplementedError("TCL configured without threads")
+                with self._tcl_lock:
+                    result = tklib.Tcl_DoOneEvent(tklib.TCL_DONT_WAIT)
+                if result == 0:
+                    time.sleep(self._busywaitinterval)
 
             if result < 0:
                 break
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -28,9 +28,11 @@
         return result
 
     elif value.typePtr == typeCache.BooleanType:
-        return result
+        return bool(value.internalRep.longValue)
     elif value.typePtr == typeCache.ByteArrayType:
-        return result
+        size = tkffi.new('int*')
+        data = tklib.Tcl_GetByteArrayFromObj(value, size)
+        return tkffi.buffer(data, size[0])[:]
     elif value.typePtr == typeCache.DoubleType:
         return value.internalRep.doubleValue
     elif value.typePtr == typeCache.IntType:
@@ -50,7 +52,7 @@
             result.append(FromObj(app, tcl_elem[0]))
         return tuple(result)
     elif value.typePtr == typeCache.ProcBodyType:
-        return result
+        pass  # fall through and return tcl object.
     elif value.typePtr == typeCache.StringType:
         buf = tklib.Tcl_GetUnicode(value)
         length = tklib.Tcl_GetCharLength(value)
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -1,6 +1,7 @@
 # C bindings with libtcl and libtk.
 
 from cffi import FFI
+import sys
 
 tkffi = FFI()
 
@@ -18,6 +19,8 @@
 #define TCL_EVAL_DIRECT ...
 #define TCL_EVAL_GLOBAL ...
 
+#define TCL_DONT_WAIT ...
+
 typedef unsigned short Tcl_UniChar;
 typedef ... Tcl_Interp;
 typedef ...* Tcl_ThreadId;
@@ -69,6 +72,7 @@
 int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
 char *Tcl_GetString(Tcl_Obj* objPtr);
 char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+unsigned char *Tcl_GetByteArrayFromObj(Tcl_Obj* objPtr, int* lengthPtr);
 
 Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
 int Tcl_GetCharLength(Tcl_Obj* objPtr);
@@ -102,6 +106,17 @@
 int Tk_GetNumMainWindows();
 """)
 
+# XXX find a better way to detect paths
+# XXX pick up CPPFLAGS and LDFLAGS and add to these paths?
+if sys.platform.startswith("openbsd"):
+    incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include']
+    linklibs = ['tk85', 'tcl85']
+    libdirs = ['/usr/local/lib', '/usr/X11R6/lib']
+else:
+    incdirs=['/usr/include/tcl']
+    linklibs=['tcl', 'tk']
+    libdirs = []
+
 tklib = tkffi.verify("""
 #include <tcl.h>
 #include <tk.h>
@@ -109,6 +124,7 @@
 char *get_tk_version() { return TK_VERSION; }
 char *get_tcl_version() { return TCL_VERSION; }
 """,
-include_dirs=['/usr/include/tcl'],
-libraries=['tcl', 'tk'],
+include_dirs=incdirs,
+libraries=linklibs,
+library_dirs = libdirs
 )
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.7"
-__version_info__ = (0, 7)
+__version__ = "0.7.2"
+__version_info__ = (0, 7, 2)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -54,7 +54,8 @@
             # _cffi_backend.so compiled.
             import _cffi_backend as backend
             from . import __version__
-            assert backend.__version__ == __version__
+            assert (backend.__version__ == __version__ or
+                    backend.__version__ == __version__[:3])
             # (If you insist you can also try to pass the option
             # 'backend=backend_ctypes.CTypesBackend()', but don't
             # rely on it!  It's probably not going to work well.)
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -30,7 +30,9 @@
         elif result in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
             result = model.PrimitiveType(result)
         else:
-            assert commontype != result
+            if commontype == result:
+                raise api.FFIError("Unsupported type: %r.  Please file a bug "
+                                   "if you think it should be." % (commontype,))
             result = resolve_common_type(result)   # recursively
         assert isinstance(result, model.BaseTypeByIdentity)
         _CACHE[commontype] = result
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -290,13 +290,26 @@
                 # assume a primitive type.  get it from .names, but reduce
                 # synonyms to a single chosen combination
                 names = list(type.names)
-                if names == ['signed'] or names == ['unsigned']:
-                    names.append('int')
-                if names[0] == 'signed' and names != ['signed', 'char']:
-                    names.pop(0)
-                if (len(names) > 1 and names[-1] == 'int'
-                        and names != ['unsigned', 'int']):
-                    names.pop()
+                if names != ['signed', 'char']:    # keep this unmodified
+                    prefixes = {}
+                    while names:
+                        name = names[0]
+                        if name in ('short', 'long', 'signed', 'unsigned'):
+                            prefixes[name] = prefixes.get(name, 0) + 1
+                            del names[0]
+                        else:
+                            break
+                    # ignore the 'signed' prefix below, and reorder the others
+                    newnames = []
+                    for prefix in ('unsigned', 'short', 'long'):
+                        for i in range(prefixes.get(prefix, 0)):
+                            newnames.append(prefix)
+                    if not names:
+                        names = ['int']    # implicitly
+                    if names == ['int']:   # but kill it if 'short' or 'long'
+                        if 'short' in prefixes or 'long' in prefixes:
+                            names = []
+                    names = newnames + names
                 ident = ' '.join(names)
                 if ident == 'void':
                     return model.void_type
@@ -500,8 +513,8 @@
                 self._partial_length = True
                 return None
         #
-        raise api.FFIError("unsupported non-constant or "
-                           "not immediately constant expression")
+        raise api.FFIError("unsupported expression: expected a "
+                           "simple numeric constant")
 
     def _build_enum_type(self, explicit_name, decls):
         if decls is not None:
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -61,7 +61,9 @@
     def load_library(self):
         # import it with the CFFI backend
         backend = self.ffi._backend
-        module = backend.load_library(self.verifier.modulefilename)
+        # needs to make a path that contains '/', on Posix
+        filename = os.path.join(os.curdir, self.verifier.modulefilename)
+        module = backend.load_library(filename)
         #
         # call loading_gen_struct() to get the struct layout inferred by
         # the C compiler
diff --git a/lib_pypy/numpy.py b/lib_pypy/numpy.py
deleted file mode 100644
--- a/lib_pypy/numpy.py
+++ /dev/null
@@ -1,5 +0,0 @@
-raise ImportError(
-    "The 'numpy' module of PyPy is in-development and not complete. "
-    "To try it out anyway, you can either import from 'numpypy', "
-    "or just write 'import numpypy' first in your program and then "
-    "import from 'numpy' as usual.")
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -6,9 +6,46 @@
 from __builtin__ import bool, int, long, float, complex, object, unicode, str
 from core import abs, max, min
 
-__all__ = []
+__version__ = '1.7.0'
+
+import os
+def get_include():
+    """
+    Return the directory that contains the NumPy \\*.h header files.
+
+    Extension modules that need to compile against NumPy should use this
+    function to locate the appropriate include directory.
+
+    Notes
+    -----
+    When using ``distutils``, for example in ``setup.py``.
+    ::
+
+        import numpy as np
+        ...
+        Extension('extension_name', ...
+                include_dirs=[np.get_include()])
+        ...
+
+    """
+    try:
+        import numpy
+    except:
+        # running from pypy source directory
+        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
+        return os.path.join(head, '../include')
+    else:
+        # using installed numpy core headers
+        import numpy.core as core
+        d = os.path.join(os.path.dirname(core.__file__), 'include')
+    return d
+
+
+
+__all__ = ['__version__', 'get_include']
 __all__ += core.__all__
 __all__ += lib.__all__
+#import sys
+#sys.modules.setdefault('numpy', sys.modules['numpypy'])
 
-import sys
-sys.modules.setdefault('numpy', sys.modules['numpypy'])
+
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
--- a/lib_pypy/numpypy/core/__init__.py
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -1,12 +1,17 @@
-import numeric
-from numeric import *
-import fromnumeric
-from fromnumeric import *
-import shape_base
-from shape_base import *
+from __future__ import division, absolute_import, print_function
 
-from fromnumeric import amax as max, amin as min
-from numeric import absolute as abs
+from . import multiarray
+from . import umath
+from . import numeric
+from .numeric import *
+from . import fromnumeric
+from .fromnumeric import *
+from . import shape_base
+from .shape_base import *
+
+from .fromnumeric import amax as max, amin as min, \
+    round_ as round
+from .numeric import absolute as abs
 
 __all__ = []
 __all__ += numeric.__all__
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -1,36 +1,53 @@
-######################################################################    
+######################################################################
 # This is a copy of numpy/core/fromnumeric.py modified for numpypy
 ######################################################################
-# Each name in __all__ was a function in  'numeric' that is now 
-# a method in 'numpy'.
-# When the corresponding method is added to numpypy BaseArray
-# each function should be added as a module function 
-# at the applevel 
-# This can be as simple as doing the following
-#
-# def func(a, ...):
-#     if not hasattr(a, 'func')
-#         a = numpypy.array(a)
-#     return a.func(...)
-#
-######################################################################
-
-import numpypy
-import _numpypy
-
-# Module containing non-deprecated functions borrowed from Numeric.
-__docformat__ = "restructuredtext en"
+"""Module containing non-deprecated functions borrowed from Numeric.
+
+"""
+from __future__ import division, absolute_import, print_function
+
+import types
+
+from . import multiarray as mu
+from . import umath as um
+from . import numerictypes as nt
+from .numeric import asarray, array, asanyarray, concatenate
+from . import _methods
+
+
+# functions that are methods
+__all__ = [
+        'alen', 'all', 'alltrue', 'amax', 'amin', 'any', 'argmax',
+        'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip',
+        'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean',
+        'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put',
+        'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_',
+        'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze',
+        'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var',
+        ]
+
+
+try:
+    _gentype = types.GeneratorType
+except AttributeError:
+    _gentype = type(None)
+
+# save away Python sum
+_sum_ = sum
 
 # functions that are now methods
-__all__ = ['take', 'reshape', 'choose', 'repeat', 'put',
-           'swapaxes', 'transpose', 'sort', 'argsort', 'argmax', 'argmin',
-           'searchsorted', 'alen',
-           'resize', 'diagonal', 'trace', 'ravel', 'nonzero', 'shape',
-           'compress', 'clip', 'sum', 'product', 'prod', 'sometrue', 'alltrue',
-           'any', 'all', 'cumsum', 'cumproduct', 'cumprod', 'ptp', 'ndim',
-           'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze',
-           'amax', 'amin',
-          ]
+def _wrapit(obj, method, *args, **kwds):
+    try:
+        wrap = obj.__array_wrap__
+    except AttributeError:
+        wrap = None
+    result = getattr(asarray(obj), method)(*args, **kwds)
+    if wrap:
+        if not isinstance(result, mu.ndarray):
+            result = asarray(result)
+        result = wrap(result)
+    return result
+
 
 def take(a, indices, axis=None, out=None, mode='raise'):
     """
@@ -46,6 +63,10 @@
         The source array.
     indices : array_like
         The indices of the values to extract.
+
+        .. versionadded:: 1.8.0
+
+        Also allow scalars for indices.
     axis : int, optional
         The axis over which to select values. By default, the flattened
         input array is used.
@@ -85,8 +106,17 @@
     >>> a[indices]
     array([4, 3, 6])
 
+    If `indices` is not one dimensional, the output also has these dimensions.
+
+    >>> np.take(a, [[0, 1], [2, 3]])
+    array([[4, 3],
+           [5, 7]])
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        take = a.take
+    except AttributeError:
+        return _wrapit(a, 'take', indices, axis, out, mode)
+    return take(indices, axis, out, mode)
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -104,16 +134,23 @@
         One shape dimension can be -1. In this case, the value is inferred
         from the length of the array and remaining dimensions.
     order : {'C', 'F', 'A'}, optional
-        Determines whether the array data should be viewed as in C
-        (row-major) order, FORTRAN (column-major) order, or the C/FORTRAN
-        order should be preserved.
+        Read the elements of `a` using this index order, and place the elements
+        into the reshaped array using this index order.  'C' means to
+        read / write the elements using C-like index order, with the last axis index
+        changing fastest, back to the first axis index changing slowest.  'F'
+        means to read / write the elements using Fortran-like index order, with
+        the first index changing fastest, and the last index changing slowest.
+        Note that the 'C' and 'F' options take no account of the memory layout
+        of the underlying array, and only refer to the order of indexing.  'A'
+        means to read / write the elements in Fortran-like index order if `a` is
+        Fortran *contiguous* in memory, C-like order otherwise.
 
     Returns
     -------
     reshaped_array : ndarray
         This will be a new view object if possible; otherwise, it will
-        be a copy.
-
+        be a copy.  Note there is no guarantee of the *memory layout* (C- or
+        Fortran- contiguous) of the returned array.
 
     See Also
     --------
@@ -121,7 +158,6 @@
 
     Notes
     -----
-
     It is not always possible to change the shape of an array without
     copying the data. If you want an error to be raise if the data is copied,
     you should assign the new shape to the shape attribute of the array::
@@ -129,12 +165,39 @@
      >>> a = np.zeros((10, 2))
      # A transpose make the array non-contiguous
      >>> b = a.T
-     # Taking a view makes it possible to modify the shape without modiying the
+     # Taking a view makes it possible to modify the shape without modifying the
      # initial object.
      >>> c = b.view()
      >>> c.shape = (20)
      AttributeError: incompatible shape for a non-contiguous array
 
+    The `order` keyword gives the index ordering both for *fetching* the values
+    from `a`, and then *placing* the values into the output array.  For example,
+    let's say you have an array:
+
+    >>> a = np.arange(6).reshape((3, 2))
+    >>> a
+    array([[0, 1],
+           [2, 3],
+           [4, 5]])
+
+    You can think of reshaping as first raveling the array (using the given
+    index order), then inserting the elements from the raveled array into the
+    new array using the same kind of index ordering as was used for the
+    raveling.
+
+    >>> np.reshape(a, (2, 3)) # C-like index ordering
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(a, (2, 3), order='F') # Fortran-like index ordering
+    array([[0, 4, 3],
+           [2, 1, 5]])
+    >>> np.reshape(np.ravel(a, order='F'), (2, 3), order='F')
+    array([[0, 4, 3],
+           [2, 1, 5]])
 
     Examples
     --------
@@ -148,12 +211,13 @@
     array([[1, 2],
            [3, 4],
            [5, 6]])
-
     """
     assert order == 'C'
-    if not hasattr(a, 'reshape'):
-       a = numpypy.array(a)
-    return a.reshape(newshape)
+    try:
+        reshape = a.reshape
+    except AttributeError:
+        return _wrapit(a, 'reshape', newshape)
+    return reshape(newshape)
 
 
 def choose(a, choices, out=None, mode='raise'):
@@ -275,7 +339,11 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    return _numpypy.choose(a, choices, out, mode)
+    try:
+        choose = a.choose
+    except AttributeError:
+        return _wrapit(a, 'choose', choices, out=out, mode=mode)
+    return choose(choices, out=out, mode=mode)
 
 
 def repeat(a, repeats, axis=None):
@@ -317,7 +385,11 @@
            [3, 4]])
 
     """
-    return _numpypy.repeat(a, repeats, axis)
+    try:
+        repeat = a.repeat
+    except AttributeError:
+        return _wrapit(a, 'repeat', repeats, axis)
+    return repeat(repeats, axis)
 
 
 def put(a, ind, v, mode='raise'):
@@ -368,7 +440,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return a.put(ind, v, mode)
 
 
 def swapaxes(a, axis1, axis2):
@@ -412,7 +484,10 @@
             [3, 7]]])
 
     """
-    swapaxes = a.swapaxes
+    try:
+        swapaxes = a.swapaxes
+    except AttributeError:
+        return _wrapit(a, 'swapaxes', axis1, axis2)
     return swapaxes(axis1, axis2)
 
 
@@ -456,9 +531,158 @@
     """
     if axes is not None:
         raise NotImplementedError('No "axes" arg yet.')
-    if not hasattr(a, 'T'):
-        a = numpypy.array(a)
-    return a.T
+    try:
+        transpose = a.transpose
+    except AttributeError:
+        return _wrapit(a, 'transpose')
+    return transpose()
+
+
+def partition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Return a partitioned copy of an array.
+
+    Creates a copy of the array with its elements rearranged in such a way that
+    the value of the element in kth position is in the position it would be in
+    a sorted array. All elements smaller than the kth element are moved before
+    this element and all equal or greater are moved behind it. The ordering of
+    the elements in the two partitions is undefined.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to be sorted.
+    kth : int or sequence of ints
+        Element index to partition by. The kth value of the element will be in
+        its final sorted position and all smaller elements will be moved before
+        it and all equal or greater elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all elements
+        indexed by kth  of them into their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort. If None, the array is flattened before
+        sorting. The default is -1, which sorts along the last axis.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'.
+    order : list, optional
+        When `a` is a structured array, this argument specifies which fields
+        to compare first, second, and so on.  This list does not need to
+        include all of the fields.
+
+    Returns
+    -------
+    partitioned_array : ndarray
+        Array of the same type and shape as `a`.
+
+    See Also
+    --------
+    ndarray.partition : Method to sort an array in-place.
+    argpartition : Indirect partition.
+    sort : Full sorting
+
+    Notes
+    -----
+    The various selection algorithms are characterized by their average speed,
+    worst case performance, work space size, and whether they are stable. A
+    stable sort keeps items with the same key in the same relative order. The
+    three available algorithms have the following properties:
+
+    ================= ======= ============= ============ =======
+       kind            speed   worst case    work space  stable
+    ================= ======= ============= ============ =======
+    'introselect'        1        O(n)           0         no
+    ================= ======= ============= ============ =======
+
+    All the partition algorithms make temporary copies of the data when
+    partitioning along any but the last axis.  Consequently, partitioning
+    along the last axis is faster and uses less space than partitioning
+    along any other axis.
+
+    The sort order for complex numbers is lexicographic. If both the real
+    and imaginary parts are non-nan then the order is determined by the
+    real parts except when they are equal, in which case the order is
+    determined by the imaginary parts.
+
+    Examples
+    --------
+    >>> a = np.array([3, 4, 2, 1])
+    >>> np.partition(a, 3)
+    array([2, 1, 3, 4])
+
+    >>> np.partition(a, (1, 3))
+    array([1, 2, 3, 4])
+
+    """
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.partition(kth, axis=axis, kind=kind, order=order)
+    return a
+
+
+def argpartition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Perform an indirect partition along the given axis using the algorithm
+    specified by the `kind` keyword. It returns an array of indices of the
+    same shape as `a` that index data along the given axis in partitioned
+    order.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to sort.
+    kth : int or sequence of ints
+        Element index to partition by. The kth element will be in its final
+        sorted position and all smaller elements will be moved before it and
+        all larger elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all of them into
+        their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort.  The default is -1 (the last axis). If None,
+        the flattened array is used.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'
+    order : list, optional
+        When `a` is an array with fields defined, this argument specifies
+        which fields to compare first, second, etc.  Not all fields need be
+        specified.
+
+    Returns
+    -------
+    index_array : ndarray, int
+        Array of indices that partition `a` along the specified axis.
+        In other words, ``a[index_array]`` yields a sorted `a`.
+
+    See Also
+    --------
+    partition : Describes partition algorithms used.
+    ndarray.partition : Inplace partition.
+    argsort : Full indirect sort
+
+    Notes
+    -----
+    See `partition` for notes on the different selection algorithms.
+
+    Examples
+    --------
+    One dimensional array:
+
+    >>> x = np.array([3, 4, 2, 1])
+    >>> x[np.argpartition(x, 3)]
+    array([2, 1, 3, 4])
+    >>> x[np.argpartition(x, (1, 3))]
+    array([1, 2, 3, 4])
+
+    """
+    return a.argpartition(kth, axis, kind=kind, order=order)
+
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -489,6 +713,7 @@
     argsort : Indirect sort.
     lexsort : Indirect stable sort on multiple keys.
     searchsorted : Find elements in a sorted array.
+    partition : Partial sort.
 
     Notes
     -----
@@ -559,7 +784,13 @@
           dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.sort(axis, kind, order)
+    return a
 
 
 def argsort(a, axis=-1, kind='quicksort', order=None):
@@ -595,6 +826,7 @@
     sort : Describes sorting algorithms used.
     lexsort : Indirect stable sort with multiple keys.
     ndarray.sort : Inplace sort.
+    argpartition : Indirect partial sort.
 
     Notes
     -----
@@ -640,7 +872,11 @@
     array([0, 1])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        argsort = a.argsort
+    except AttributeError:
+        return _wrapit(a, 'argsort', axis, kind, order)
+    return argsort(axis, kind, order)
 
 
 def argmax(a, axis=None):
@@ -694,9 +930,11 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmax'):
-        a = numpypy.array(a)
-    return a.argmax()
+    try:
+        argmax = a.argmax
+    except AttributeError:
+        return _wrapit(a, 'argmax')
+    return argmax()
 
 
 def argmin(a, axis=None):
@@ -710,29 +948,37 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmin'):
-        a = numpypy.array(a)
-    return a.argmin()
-
-
-def searchsorted(a, v, side='left'):
+    try:
+        argmin = a.argmin
+    except AttributeError:
+        return _wrapit(a, 'argmin')
+    return argmin()
+
+
+def searchsorted(a, v, side='left', sorter=None):
     """
     Find indices where elements should be inserted to maintain order.
 
-    Find the indices into a sorted array `a` such that, if the corresponding
-    elements in `v` were inserted before the indices, the order of `a` would
-    be preserved.
+    Find the indices into a sorted array `a` such that, if the
+    corresponding elements in `v` were inserted before the indices, the
+    order of `a` would be preserved.
 
     Parameters
     ----------
     a : 1-D array_like
-        Input array, sorted in ascending order.
+        Input array. If `sorter` is None, then it must be sorted in
+        ascending order, otherwise `sorter` must be an array of indices
+        that sort it.
     v : array_like
         Values to insert into `a`.
     side : {'left', 'right'}, optional
-        If 'left', the index of the first suitable location found is given.  If
-        'right', return the last such index.  If there is no suitable
+        If 'left', the index of the first suitable location found is given.
+        If 'right', return the last such index.  If there is no suitable
         index, return either 0 or N (where N is the length of `a`).
+    sorter : 1-D array_like, optional
+        .. versionadded:: 1.7.0
+        Optional array of integer indices that sort array a into ascending
+        order. They are typically the result of argsort.
 
     Returns
     -------
@@ -761,7 +1007,11 @@
     array([0, 5, 1, 2])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        searchsorted = a.searchsorted
+    except AttributeError:
+        return _wrapit(a, 'searchsorted', v, side, sorter)
+    return searchsorted(v, side, sorter)
 
 
 def resize(a, new_shape):
@@ -802,10 +1052,30 @@
            [0, 1, 2, 3]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def squeeze(a):
+    if isinstance(new_shape, (int, nt.integer)):
+        new_shape = (new_shape,)
+    a = ravel(a)
+    Na = len(a)
+    if not Na: return mu.zeros(new_shape, a.dtype.char)
+    total_size = um.multiply.reduce(new_shape)
+    n_copies = int(total_size / Na)
+    extra = total_size % Na
+
+    if total_size == 0:
+        return a[:0]
+
+    if extra != 0:
+        n_copies = n_copies+1
+        extra = Na-extra
+
+    a = concatenate( (a,)*n_copies)
+    if extra > 0:
+        a = a[:-extra]
+
+    return reshape(a, new_shape)
+
+
+def squeeze(a, axis=None):
     """
     Remove single-dimensional entries from the shape of an array.
 
@@ -813,12 +1083,19 @@
     ----------
     a : array_like
         Input data.
+    axis : None or int or tuple of ints, optional
+        .. versionadded:: 1.7.0
+
+        Selects a subset of the single-dimensional entries in the
+        shape. If an axis is selected with shape entry greater than
+        one, an error is raised.
 
     Returns
     -------
     squeezed : ndarray
-        The input array, but with with all dimensions of length 1
-        removed.  Whenever possible, a view on `a` is returned.
+        The input array, but with with all or a subset of the
+        dimensions of length 1 removed. This is always `a` itself
+        or a view into `a`.
 
     Examples
     --------
@@ -827,9 +1104,20 @@
     (1, 3, 1)
     >>> np.squeeze(x).shape
     (3,)
+    >>> np.squeeze(x, axis=(2,)).shape
+    (1, 3)
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        squeeze = a.squeeze
+    except AttributeError:
+        return _wrapit(a, 'squeeze')
+    try:
+        # First try to use the new axis= parameter
+        return squeeze(axis=axis)
+    except TypeError:
+        # For backwards compatibility
+        return squeeze()
 
 
 def diagonal(a, offset=0, axis1=0, axis2=1):
@@ -844,6 +1132,27 @@
     removing `axis1` and `axis2` and appending an index to the right equal
     to the size of the resulting diagonals.
 
+    In versions of NumPy prior to 1.7, this function always returned a new,
+    independent array containing a copy of the values in the diagonal.
+
+    In NumPy 1.7 and 1.8, it continues to return a copy of the diagonal,
+    but depending on this fact is deprecated. Writing to the resulting
+    array continues to work as it used to, but a FutureWarning is issued.
+
+    In NumPy 1.9 it returns a read-only view on the original array.
+    Attempting to write to the resulting array will produce an error.
+
+    In NumPy 1.10, it will return a read/write view, Writing to the returned
+    array will alter your original array.
+
+    If you don't write to the array returned by this function, then you can
+    just ignore all of the above.
+
+    If you depend on the current behavior, then we suggest copying the
+    returned array explicitly, i.e., use ``np.diagonal(a).copy()`` instead of
+    just ``np.diagonal(a)``. This will work with both past and future versions
+    of NumPy.
+
     Parameters
     ----------
     a : array_like
@@ -913,7 +1222,7 @@
            [5, 7]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return asarray(a).diagonal(offset, axis1, axis2)
 
 
 def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None):
@@ -972,7 +1281,7 @@
     (2, 3)
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return asarray(a).trace(offset, axis1, axis2, dtype, out)
 
 def ravel(a, order='C'):
     """
@@ -984,21 +1293,25 @@
     Parameters
     ----------
     a : array_like
-        Input array.  The elements in ``a`` are read in the order specified by
+        Input array.  The elements in `a` are read in the order specified by
         `order`, and packed as a 1-D array.
     order : {'C','F', 'A', 'K'}, optional
-        The elements of ``a`` are read in this order. 'C' means to view
-        the elements in C (row-major) order. 'F' means to view the elements
-        in Fortran (column-major) order. 'A' means to view the elements
-        in 'F' order if a is Fortran contiguous, 'C' order otherwise.
-        'K' means to view the elements in the order they occur in memory,
-        except for reversing the data when strides are negative.
-        By default, 'C' order is used.
+        The elements of `a` are read using this index order. 'C' means to
+        index the elements in C-like order, with the last axis index changing
+        fastest, back to the first axis index changing slowest.   'F' means to
+        index the elements in Fortran-like index order, with the first index
+        changing fastest, and the last index changing slowest. Note that the 'C'
+        and 'F' options take no account of the memory layout of the underlying
+        array, and only refer to the order of axis indexing.  'A' means to read
+        the elements in Fortran-like index order if `a` is Fortran *contiguous*
+        in memory, C-like order otherwise.  'K' means to read the elements in
+        the order they occur in memory, except for reversing the data when
+        strides are negative.  By default, 'C' index order is used.
 
     Returns
     -------
     1d_array : ndarray
-        Output of the same dtype as `a`, and of shape ``(a.size(),)``.
+        Output of the same dtype as `a`, and of shape ``(a.size,)``.
 
     See Also
     --------
@@ -1008,11 +1321,11 @@
 
     Notes
     -----
-    In row-major order, the row index varies the slowest, and the column
-    index the quickest.  This can be generalized to multiple dimensions,
-    where row-major order implies that the index along the first axis
-    varies slowest, and the index along the last quickest.  The opposite holds
-    for Fortran-, or column-major, mode.
+    In C-like (row-major) order, in two dimensions, the row index varies the
+    slowest, and the column index the quickest.  This can be generalized to
+    multiple dimensions, where row-major order implies that the index along the
+    first axis varies slowest, and the index along the last quickest.  The
+    opposite holds for Fortran-like, or column-major, index ordering.
 
     Examples
     --------
@@ -1056,9 +1369,8 @@
     array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11])
 
     """
-    if not hasattr(a, 'ravel'):
-        a = numpypy.array(a)
-    return a.ravel(order=order)
+    return asarray(a).ravel(order)
+
 
 def nonzero(a):
     """
@@ -1133,7 +1445,13 @@
     (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2]))
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        nonzero = a.nonzero
+    except AttributeError:
+        res = _wrapit(a, 'nonzero')
+    else:
+        res = nonzero()
+    return res
 
 
 def shape(a):
@@ -1174,9 +1492,11 @@
     (2,)
 
     """
-    if not hasattr(a, 'shape'):
-        a = numpypy.array(a)
-    return a.shape
+    try:
+        result = a.shape
+    except AttributeError:
+        result = asarray(a).shape
+    return result
 
 
 def compress(condition, a, axis=None, out=None):
@@ -1211,7 +1531,8 @@
     See Also
     --------
     take, choose, diag, diagonal, select
-    ndarray.compress : Equivalent method.
+    ndarray.compress : Equivalent method in ndarray
+    np.extract: Equivalent method when working on 1-D arrays
     numpy.doc.ufuncs : Section "Output arguments"
 
     Examples
@@ -1238,7 +1559,11 @@
     array([2])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        compress = a.compress
+    except AttributeError:
+        return _wrapit(a, 'compress', condition, axis, out)
+    return compress(condition, axis, out)
 
 
 def clip(a, a_min, a_max, out=None):
@@ -1291,12 +1616,14 @@
     array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8])
 
     """
-    if not hasattr(a, 'clip'):
-        a = numpypy.array(a)
-    return a.clip(a_min, a_max, out=out)
-
-
-def sum(a, axis=None, dtype=None, out=None):
+    try:
+        clip = a.clip
+    except AttributeError:
+        return _wrapit(a, 'clip', a_min, a_max, out)
+    return clip(a_min, a_max, out)
+
+
+def sum(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Sum of array elements over a given axis.
 
@@ -1304,9 +1631,16 @@
     ----------
     a : array_like
         Elements to sum.
-    axis : integer, optional
-        Axis over which the sum is taken. By default `axis` is None,
-        and all elements are summed.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a sum is performed.
+        The default (`axis` = `None`) is perform a sum over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a sum is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     dtype : dtype, optional
         The type of the returned array and of the accumulator in which
         the elements are summed.  By default, the dtype of `a` is used.
@@ -1319,6 +1653,10 @@
         (the shape of `a` with `axis` removed, i.e.,
         ``numpy.delete(a.shape, axis)``).  Its type is preserved. See
         `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1362,13 +1700,25 @@
     -128
 
     """
-    assert dtype is None
-    if not hasattr(a, "sum"):
-        a = numpypy.array(a)
-    return a.sum(axis=axis, out=out)
-
-
-def product (a, axis=None, dtype=None, out=None):
+    if isinstance(a, _gentype):
+        res = _sum_(a)
+        if out is not None:
+            out[...] = res
+            return out
+        return res
+    elif type(a) is not mu.ndarray:
+        try:
+            sum = a.sum
+        except AttributeError:
+            return _methods._sum(a, axis=axis, dtype=dtype,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameters here...
+        return sum(axis=axis, dtype=dtype, out=out)
+    else:
+        return _methods._sum(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
+
+def product (a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Return the product of array elements over a given axis.
 
@@ -1377,10 +1727,10 @@
     prod : equivalent function; see for details.
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def sometrue(a, axis=None, out=None):
+    return um.multiply.reduce(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+
+def sometrue(a, axis=None, out=None, keepdims=False):
     """
     Check whether some values are true.
 
@@ -1391,14 +1741,14 @@
     any : equivalent function
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'any'):
-        a = numpypy.array(a)
-    return a.any()
-
-
-def alltrue (a, axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.any(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.any(axis=axis, out=out)
+
+def alltrue (a, axis=None, out=None, keepdims=False):
     """
     Check if all elements of input array are true.
 
@@ -1407,13 +1757,14 @@
     numpy.all : Equivalent function; see for details.
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'all'):
-        a = numpypy.array(a)
-    return a.all()
-
-def any(a,axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.all(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.all(axis=axis, out=out)
+
+def any(a, axis=None, out=None, keepdims=False):
     """
     Test whether any array element along a given axis evaluates to True.
 
@@ -1423,17 +1774,26 @@
     ----------
     a : array_like
         Input array or object that can be converted to an array.
-    axis : int, optional
-        Axis along which a logical OR is performed.  The default
-        (`axis` = `None`) is to perform a logical OR over a flattened
-        input array. `axis` may be negative, in which case it counts
-        from the last to the first axis.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical OR reduction is performed.
+        The default (`axis` = `None`) is perform a logical OR over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a reduction is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     out : ndarray, optional
         Alternate output array in which to place the result.  It must have
         the same shape as the expected output and its type is preserved
         (e.g., if it is of type float, then it will remain so, returning
         1.0 for True and 0.0 for False, regardless of the type of `a`).
         See `doc.ufuncs` (Section "Output arguments") for details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1477,14 +1837,14 @@
     (191614240, 191614240)
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'any'):
-        a = numpypy.array(a)
-    return a.any()
-
-
-def all(a,axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.any(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.any(axis=axis, out=out)
+
+def all(a, axis=None, out=None, keepdims=False):
     """
     Test whether all array elements along a given axis evaluate to True.
 
@@ -1492,17 +1852,26 @@
     ----------
     a : array_like
         Input array or object that can be converted to an array.
-    axis : int, optional
-        Axis along which a logical AND is performed.
-        The default (`axis` = `None`) is to perform a logical AND
-        over a flattened input array.  `axis` may be negative, in which
-        case it counts from the last to the first axis.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical AND reduction is performed.
+        The default (`axis` = `None`) is perform a logical OR over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a reduction is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     out : ndarray, optional
         Alternate output array in which to place the result.
         It must have the same shape as the expected output and its
         type is preserved (e.g., if ``dtype(out)`` is float, the result
         will consist of 0.0's and 1.0's).  See `doc.ufuncs` (Section
         "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1541,12 +1910,12 @@
     (28293632, 28293632, array([ True], dtype=bool))
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'all'):
-        a = numpypy.array(a)

From noreply at buildbot.pypy.org  Wed Oct 16 14:09:32 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 14:09:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: kill a test that was skipped anyway
Message-ID: <20131016120932.728F51C0203@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67415:c8e69f7a4469
Date: 2013-10-16 14:01 +0200
http://bitbucket.org/pypy/pypy/changeset/c8e69f7a4469/

Log:	kill a test that was skipped anyway

diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -649,89 +649,5 @@
         newobj1 = oldobj.next
         assert newobj1.x == 1337
 
-    # Test trying to be a bit comprehensive about
-    # states and types of objects
-    def test_allocate_states(self):
-        py.test.skip("broken test for now")
-        from rpython.memory.gc import incminimark
-        largeobj_size =  self.gc.nonlarge_max + 1
-
-        assert self.gc.gc_state == incminimark.STATE_SCANNING
-        assert self.gc.get_total_memory_used() == 0
-
-        for i in range(5):
-            curobj = self.malloc(S)
-            curobj.x = i
-            self.stackroots.append(curobj)
-            assert self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(curobj))
-
-        for i in range(5):
-            curobj = self.malloc(VAR, largeobj_size)
-            self.stackroots.append(curobj)
-            assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(curobj))
-
-        assert self.gc.gc_state == incminimark.STATE_SCANNING
-
-        self.gc.debug_gc_step()   # this reads self.stackroots
-        reachableroot = self.stackroots[4]
-
-        nallocated = {}
-
-        reachable = []
-        unreachable = []
-
-        while True:
-            
-            if self.gc.gc_state not in nallocated:
-                nallocated[self.gc.gc_state] = 0
-
-            if nallocated[self.gc.gc_state] < 1:
-                unreachableobj = self.malloc(S)
-                reachableobj = self.malloc(S)
-                assert self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(reachableobj))
-                reachableviayoungobj = self.malloc(S)
-                self.write(reachableobj,'next',reachableviayoungobj)
-                unreachableobj.x = 150
-                reachableobj.x = 150
-                reachableviayoungobj.x = 150
-
-                self.write(reachableroot,'next',reachableobj)
-                reachableroot = reachableobj
-
-                unreachable.append(unreachableobj)
-                reachable.append(reachableobj)
-                reachable.append(reachableviayoungobj)
-
-                nallocated[self.gc.gc_state] += 1
-
-            if self.gc.gc_state == incminimark.STATE_SCANNING:
-                pass
-            elif self.gc.gc_state == incminimark.STATE_MARKING:
-                pass
-            elif self.gc.gc_state == incminimark.STATE_SWEEPING_RAWMALLOC:
-                pass
-            elif self.gc.gc_state == incminimark.STATE_SWEEPING_ARENA:
-                pass
-            elif self.gc.gc_state == incminimark.STATE_FINALIZING:
-                # ASSUMPTION finalizing is atomic
-                #
-                #complete collection
-                self.gc.debug_gc_step()
-                assert self.gc.gc_state == incminimark.STATE_SCANNING
-                break
-            else:
-                raise Exception("unreachable")
-
-            self.gc.debug_gc_step()
-
-        #complete the next collection cycle
-        self.gc.debug_gc_step_until(incminimark.STATE_SCANNING)
-
-        for obj in reachable:
-            assert obj.x == 150
-
-        for obj in unreachable:
-            assert py.test.raises(RuntimeError,"obj.x")
-
 class TestIncrementalMiniMarkGCFull(DirectGCTest):
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass

From noreply at buildbot.pypy.org  Wed Oct 16 14:09:33 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 14:09:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: write a failing test
Message-ID: <20131016120933.98E5A1C0203@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67416:243756bd6001
Date: 2013-10-16 14:04 +0200
http://bitbucket.org/pypy/pypy/changeset/243756bd6001/

Log:	write a failing test

diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -10,6 +10,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.memory.gctypelayout import TypeLayoutBuilder
 from rpython.rlib.rarithmetic import LONG_BIT, is_valid_int
+from rpython.memory.gc import incminimark
 
 WORD = LONG_BIT // 8
 
@@ -513,7 +514,6 @@
     test_card_marker.GC_PARAMS = {"card_page_indices": 4}
 
     def test_writebarrier_before_copy(self):
-        from rpython.memory.gc import minimark
         largeobj_size =  self.gc.nonlarge_max + 1
         self.gc.next_major_collection_threshold = 99999.0
         p_src = self.malloc(VAR, largeobj_size)
@@ -552,7 +552,6 @@
 
     def test_writebarrier_before_copy_preserving_cards(self):
         from rpython.rtyper.lltypesystem import llarena
-        from rpython.memory.gc import minimark
         tid = self.get_type_id(VAR)
         largeobj_size =  self.gc.nonlarge_max + 1
         self.gc.next_major_collection_threshold = 99999.0
@@ -625,8 +624,6 @@
         self.gc.debug_check_consistency()
 
     def test_sweeping_simple(self):
-        from rpython.memory.gc import incminimark
-
         assert self.gc.gc_state == incminimark.STATE_SCANNING
 
         for i in range(2):
@@ -649,5 +646,18 @@
         newobj1 = oldobj.next
         assert newobj1.x == 1337
 
+    def test_obj_on_escapes_on_stack(self):
+        obj0 = self.malloc(S)
+
+        self.stackroots.append(obj0)
+        obj0.next = self.malloc(S)
+        self.gc.debug_gc_step_until(incminimark.STATE_MARKING)
+        obj1 = obj0.next
+        obj1.x = 13
+        obj0.next = lltype.nullptr(S)
+        self.stackroots.append(obj1)
+        self.gc.debug_gc_step_until(incminimark.STATE_SCANNING)
+        assert self.stackroots[0].x == 13
+
 class TestIncrementalMiniMarkGCFull(DirectGCTest):
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass

From noreply at buildbot.pypy.org  Wed Oct 16 14:09:34 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 14:09:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix
Message-ID: <20131016120934.C45561C0203@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67417:bc13dac86489
Date: 2013-10-16 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/bc13dac86489/

Log:	fix

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1597,6 +1597,9 @@
             if (bool(self.young_rawmalloced_objects)
                 and self.young_rawmalloced_objects.contains(obj)):
                 self._visit_young_rawmalloced_object(obj)
+            if self.gc_state == STATE_MARKING:
+                if not self.header(obj).tid & GCFLAG_VISITED:
+                    self.objects_to_trace.append(obj)
             return
         #
         size_gc_header = self.gcheaderbuilder.size_gc_header
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -652,12 +652,13 @@
         self.stackroots.append(obj0)
         obj0.next = self.malloc(S)
         self.gc.debug_gc_step_until(incminimark.STATE_MARKING)
+        obj0 = self.stackroots[-1]
         obj1 = obj0.next
         obj1.x = 13
         obj0.next = lltype.nullptr(S)
         self.stackroots.append(obj1)
         self.gc.debug_gc_step_until(incminimark.STATE_SCANNING)
-        assert self.stackroots[0].x == 13
+        assert self.stackroots[1].x == 13
 
 class TestIncrementalMiniMarkGCFull(DirectGCTest):
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass

From noreply at buildbot.pypy.org  Wed Oct 16 14:17:43 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 14:17:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Tweak: don't decrement the amount of
 work left when we see
Message-ID: <20131016121743.49B0C1C0203@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67418:c86aa822e076
Date: 2013-10-16 14:17 +0200
http://bitbucket.org/pypy/pypy/changeset/c86aa822e076/

Log:	Tweak: don't decrement the amount of work left when we see an
	already-visited object.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1981,12 +1981,9 @@
     def visit_all_objects_step(self, size_to_track):
         # Objects can be added to pending by visit
         pending = self.objects_to_trace
-        size_gc_header = self.gcheaderbuilder.size_gc_header
         while size_to_track > 0 and pending.non_empty():
             obj = pending.pop()
-            self.visit(obj)
-            totalsize = size_gc_header + self.get_size(obj)
-            size_to_track -= raw_malloc_usage(totalsize)
+            size_to_track = self.visit(obj)
 
     def visit(self, obj):
         #
@@ -2001,19 +1998,21 @@
         # collection.
         hdr = self.header(obj)
         if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
-            return
+            return 0
         #
         # It's the first time.  We set the flag VISITED.  The trick is
         # to also set TRACK_YOUNG_PTRS here, for the write barrier.
         hdr.tid |= GCFLAG_VISITED | GCFLAG_TRACK_YOUNG_PTRS
 
-        if not self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)):
-            return
-        #
-        # Trace the content of the object and put all objects it references
-        # into the 'objects_to_trace' list.
-        self.trace(obj, self._collect_ref_rec, None)
+        if self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)):
+            #
+            # Trace the content of the object and put all objects it references
+            # into the 'objects_to_trace' list.
+            self.trace(obj, self._collect_ref_rec, None)
 
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        totalsize = size_gc_header + self.get_size(obj)
+        return raw_malloc_usage(totalsize)
 
     # ----------
     # id() and identityhash() support

From noreply at buildbot.pypy.org  Wed Oct 16 14:44:49 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 14:44:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: Load the typeids.txt from the
	executable directly
Message-ID: <20131016124449.C680F1C011A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67419:3eb47e8d1038
Date: 2013-10-16 12:44 +0000
http://bitbucket.org/pypy/pypy/changeset/3eb47e8d1038/

Log:	Load the typeids.txt from the executable directly

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -113,22 +113,18 @@
             self.prog2typeids[progspace] = typeids
             return typeids
 
-    def load_typeids(self, progspace):
+    def load_typeids(self, progspace=None):
         """
         Returns a mapping offset --> description
         """
-        exename = getattr(progspace, 'filename', '')
-        root = os.path.dirname(exename)
-        # XXX The same information is found in
-        # XXX   pypy_g_rpython_memory_gctypelayout_GCData.gcd_inst_typeids_z
-        # XXX Find out how to read it
-        typeids_txt = os.path.join(root, 'typeids.txt')
-        if not os.path.exists(typeids_txt):
-            newroot = os.path.dirname(root)
-            typeids_txt = os.path.join(newroot, 'typeids.txt')
-        print 'loading', typeids_txt
-        with open(typeids_txt) as f:
-            typeids = TypeIdsMap(f.readlines(), self.gdb)
+        vname = 'pypy_g_rpython_memory_gctypelayout_GCData.gcd_inst_typeids_z'
+        length = int(self.gdb.parse_and_eval('*(long*)%s' % vname))
+        vstart = '(char*)(((long*)%s)+1)' % vname
+        self.gdb.execute('dump binary memory /tmp/typeids.txt.z %s %s+%d'
+                         % (vstart, vstart, length))
+        s = open('/tmp/typeids.txt.z', 'rb').read()
+        import zlib; typeids_txt = zlib.decompress(s)
+        typeids = TypeIdsMap(typeids_txt.splitlines(True), self.gdb)
         return typeids
 
 

From noreply at buildbot.pypy.org  Wed Oct 16 14:52:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 14:52:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: Remove this outdated comment.
Message-ID: <20131016125223.173481C00ED@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67420:7a8acdebd959
Date: 2013-10-16 14:51 +0200
http://bitbucket.org/pypy/pypy/changeset/7a8acdebd959/

Log:	Remove this outdated comment.

diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py
--- a/pypy/tool/gdb_pypy.py
+++ b/pypy/tool/gdb_pypy.py
@@ -58,7 +58,6 @@
 class RPyType(Command):
     """
     Prints the RPython type of the expression (remember to dereference it!)
-    It assumes to find ``typeids.txt`` in the current directory.
     E.g.:
 
     (gdb) rpy_type *l_v123

From noreply at buildbot.pypy.org  Wed Oct 16 15:17:10 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 15:17:10 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: (fijal,
	agaynor) write rebuildingresumebuilder
Message-ID: <20131016131710.799B41C011A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67421:4efa7708d887
Date: 2013-10-16 15:10 +0200
http://bitbucket.org/pypy/pypy/changeset/4efa7708d887/

Log:	(fijal, agaynor) write rebuildingresumebuilder

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -9,6 +9,7 @@
                                          BoxFloat, ConstFloat)
 from rpython.jit.metainterp.resoperation import ResOperation, rop
 from rpython.jit.metainterp.typesystem import deref
+from rpython.jit.metainterp.resume2 import rebuild_faillocs_from_resumedata
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.tool.oparser import parse
diff --git a/rpython/jit/metainterp/resume2.py b/rpython/jit/metainterp/resume2.py
--- a/rpython/jit/metainterp/resume2.py
+++ b/rpython/jit/metainterp/resume2.py
@@ -14,7 +14,7 @@
     def rebuild(self, faildescr):
         self._rebuild_until(faildescr.rd_resume_bytecode,
                             faildescr.rd_bytecode_position)
-        self.finish()
+        return self.finish()
 
     def finish(self):
         pass
@@ -96,3 +96,4 @@
 
 def rebuild_from_resumedata(metainterp, deadframe, faildescr):
     BoxResumeReader(metainterp, deadframe).rebuild(faildescr)
+
diff --git a/rpython/jit/metainterp/test/test_resume2.py b/rpython/jit/metainterp/test/test_resume2.py
--- a/rpython/jit/metainterp/test/test_resume2.py
+++ b/rpython/jit/metainterp/test/test_resume2.py
@@ -1,9 +1,10 @@
 
+import py
 from rpython.jit.tool.oparser import parse
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.metainterp.history import AbstractDescr
 from rpython.jit.metainterp.resume2 import rebuild_from_resumedata,\
-     ResumeBytecode
+     ResumeBytecode, BoxResumeReader
 
 
 class Descr(AbstractDescr):
@@ -16,10 +17,17 @@
     def __init__(self, jitcode):
         self.jitcode = jitcode
         self.registers_i = [None] * jitcode.num_regs_i()
+        self.registers_r = [None] * jitcode.num_regs_r()
+        self.registers_f = [None] * jitcode.num_regs_f()
 
     def num_nonempty_regs(self):
         return len(filter(bool, self.registers_i))
 
+    def dump_registers(self, lst, backend_values):
+        lst += [backend_values[x] for x in self.registers_i]
+        lst += [backend_values[x] for x in self.registers_r]
+        lst += [backend_values[x] for x in self.registers_f]        
+
 class MockMetaInterp(object):
     def __init__(self):
         self.framestack = []
@@ -35,6 +43,20 @@
         assert frame == "myframe"
         return index + 3
 
+class RebuildingResumeReader(BoxResumeReader):
+    def __init__(self):
+        self.backend_values = {}
+        self.metainterp = MockMetaInterp()
+
+    def finish(self):
+        l = []
+        for frame in self.metainterp.framestack:
+            frame.dump_registers(l, self.backend_values)
+        return l
+
+def rebuild_locs_from_resumedata(faildescr):
+    return RebuildingResumeReader().rebuild(faildescr)
+
 class TestResumeDirect(object):
     def test_box_resume_reader(self):
         jitcode = JitCode("jitcode")
@@ -133,6 +155,7 @@
         assert f.registers_i[1].getint() == 2 + 3
 
     def test_new(self):
+        py.test.skip("finish")
         jitcode1 = JitCode("jitcode")
         jitcode1.setup(num_regs_i=1)
         base = parse("""
@@ -142,28 +165,29 @@
         resume_setfield(i0, 13, descr=fielddescr)
         backend_put(12,
         leave_frame()
-        """, namespace={'jitcode':jitcode})
+        """, namespace={'jitcode':jitcode1})
 
     def test_reconstructing_resume_reader(self):
-        XXX
         jitcode1 = JitCode("jitcode")
-        jitcode1.setup(num_regs_i=3, num_regs_f=0, num_regs_r=0)
+        jitcode1.setup(num_regs_i=2, num_regs_f=0, num_regs_r=0)
         jitcode2 = JitCode("jitcode2")
-        jitcode2.setup(num_regs_i=3, num_regs_f=0, num_regs_r=0)
+        jitcode2.setup(num_regs_i=1, num_regs_f=0, num_regs_r=0)
         resume_loop = parse("""
-        []
+        [i0, i1, i2, i3]
         enter_frame(-1, descr=jitcode1)
-        backend_put(11, 0, 1)
+        resume_put(i0, 0, 1)
+        backend_attach(i0, 11)
         enter_frame(12, descr=jitcode2)
-        backend_put(12, 0, 2)
-        backend_put(8, 1, 0)
+        resume_put(i1, 0, 0)
+        backend_attach(i1, 12)
+        resume_put(i3, 1, 0)
+        backend_attach(i3, 8)
         leave_frame()
-        backend_put(10, 0, 0)
         leave_frame()
         """, namespace={'jitcode1': jitcode1,
                         'jitcode2': jitcode2})
         descr = Descr()
         descr.rd_resume_bytecode = ResumeBytecode(resume_loop.operations)
-        descr.rd_bytecode_position = 5
+        descr.rd_bytecode_position = 8
         locs = rebuild_locs_from_resumedata(descr)
-        assert locs == [8, 11, -1, -1, -1, 12]
+        assert locs == [8, 11, 12]

From noreply at buildbot.pypy.org  Wed Oct 16 15:17:11 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 15:17:11 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: (fijal,
	agaynor) rebuild faillocs for the bridge
Message-ID: <20131016131711.BC5291C011A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67422:8604e65fa007
Date: 2013-10-16 15:16 +0200
http://bitbucket.org/pypy/pypy/changeset/8604e65fa007/

Log:	(fijal, agaynor) rebuild faillocs for the bridge

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -111,9 +111,7 @@
         self._debug = v
         return r
 
-    def rebuild_faillocs_from_descr(self, descr, inputargs):
-        XXX
-        loc_positions = rebuild_locs_from_resumedata(descr)
+    def rebuild_faillocs_from_descr(self, descr, inputargs, loc_positions):
         locs = [None] * len(loc_positions)
         GPR_REGS = len(self.cpu.gen_regs)
         XMM_REGS = len(self.cpu.float_regs)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -68,8 +68,8 @@
         """
         raise NotImplementedError
 
-    def compile_bridge(self, logger, faildescr, inputargs, operations,
-                       original_loop_token, log=True):
+    def compile_bridge(self, logger, faildescr, inputargs, backend_positions,
+                       operations, original_loop_token, log=True):
         """Assemble the bridge.
         The FailDescr is the descr of the original guard that failed.
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -9,7 +9,7 @@
                                          BoxFloat, ConstFloat)
 from rpython.jit.metainterp.resoperation import ResOperation, rop
 from rpython.jit.metainterp.typesystem import deref
-from rpython.jit.metainterp.resume2 import rebuild_faillocs_from_resumedata
+from rpython.jit.metainterp.test.test_resume2 import rebuild_locs_from_resumedata
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.tool.oparser import parse
@@ -246,7 +246,8 @@
             ResOperation(rop.JUMP, [i1b], None, descr=targettoken),
         ]
 
-        self.cpu.compile_bridge(None, faildescr1, [i1b], bridge, looptoken)
+        locs = rebuild_locs_from_resumedata(faildescr1)
+        self.cpu.compile_bridge(None, faildescr1, [i1b], locs, bridge, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -514,8 +514,8 @@
         return AsmInfo(ops_offset, rawstart + looppos,
                        size_excluding_failure_stuff - looppos)
 
-    def assemble_bridge(self, logger, faildescr, inputargs, operations,
-                        original_loop_token, log):
+    def assemble_bridge(self, logger, faildescr, inputargs, backend_positions,
+                        operations, original_loop_token, log):
         if not we_are_translated():
             # Arguments should be unique
             assert len(set(inputargs)) == len(inputargs)
@@ -526,7 +526,7 @@
             operations = self._inject_debugging_code(faildescr, operations,
                                                      'b', descr_number)
 
-        arglocs = self.rebuild_faillocs_from_descr(faildescr, inputargs)
+        arglocs = self.rebuild_faillocs_from_descr(faildescr, inputargs, backend_positions)
         regalloc = RegAlloc(self, self.cpu.translate_support_code)
         startpos = self.mc.get_relative_pos()
         operations = regalloc.prepare_bridge(inputargs, arglocs,
@@ -538,7 +538,7 @@
         frame_depth_no_fixed_size = self._assemble(regalloc, inputargs, operations)
         codeendpos = self.mc.get_relative_pos()
         self.resume_bytecode = regalloc.resumebuilder.finish(
-            faildescr.rd_bytecode, original_loop_token)
+            faildescr.rd_resume_bytecode, original_loop_token)
         self.write_pending_failure_recoveries()
         fullsize = self.mc.get_relative_pos()
         #
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -93,12 +93,12 @@
         return self.assembler.assemble_loop(logger, name, inputargs, operations,
                                             looptoken, log=log)
 
-    def compile_bridge(self, logger, faildescr, inputargs, operations,
-                       original_loop_token, log=True):
+    def compile_bridge(self, logger, faildescr, inputargs, backend_positions,
+                       operations, original_loop_token, log=True):
         clt = original_loop_token.compiled_loop_token
         clt.compiling_a_bridge()
         return self.assembler.assemble_bridge(logger, faildescr, inputargs,
-                                              operations,
+                                              backend_positions, operations,
                                               original_loop_token, log=log)
 
     def clear_latest_values(self, count):

From noreply at buildbot.pypy.org  Wed Oct 16 15:45:15 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 15:45:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move the three new lines outside the
 performance-critical
Message-ID: <20131016134515.554F81C00ED@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67423:b08218207ffd
Date: 2013-10-16 15:44 +0200
http://bitbucket.org/pypy/pypy/changeset/b08218207ffd/

Log:	Move the three new lines outside the performance-critical
	_trace_drag_out().

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1580,6 +1580,14 @@
 
 
     def _trace_drag_out1(self, root):
+        # In the MARKING state, we must also record this old object,
+        # if it is not VISITED yet.
+        if self.gc_state == STATE_MARKING:
+            obj = root.address[0]
+            if not self.is_in_nursery(obj):
+                if not self.header(obj).tid & GCFLAG_VISITED:
+                    self.objects_to_trace.append(obj)
+        #
         self._trace_drag_out(root, None)
 
     def _trace_drag_out(self, root, ignored):
@@ -1597,9 +1605,6 @@
             if (bool(self.young_rawmalloced_objects)
                 and self.young_rawmalloced_objects.contains(obj)):
                 self._visit_young_rawmalloced_object(obj)
-            if self.gc_state == STATE_MARKING:
-                if not self.header(obj).tid & GCFLAG_VISITED:
-                    self.objects_to_trace.append(obj)
             return
         #
         size_gc_header = self.gcheaderbuilder.size_gc_header

From noreply at buildbot.pypy.org  Wed Oct 16 16:08:00 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 16:08:00 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: in progress
Message-ID: <20131016140800.23D731C00ED@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67424:7945084fc05d
Date: 2013-10-16 16:07 +0200
http://bitbucket.org/pypy/pypy/changeset/7945084fc05d/

Log:	in progress

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -326,6 +326,10 @@
             if self.frame_manager is not None:
                 self.frame_manager.mark_as_free(v)
 
+    def free_unused_regs(self):
+        for v in self.reg_bindings:
+            self.possibly_free_var(v)
+
     def possibly_free_vars(self, vars):
         """ Same as 'possibly_free_var', but for all v in vars.
         """
diff --git a/rpython/jit/backend/llsupport/resumebuilder.py b/rpython/jit/backend/llsupport/resumebuilder.py
--- a/rpython/jit/backend/llsupport/resumebuilder.py
+++ b/rpython/jit/backend/llsupport/resumebuilder.py
@@ -75,6 +75,6 @@
                 self._mark_visited(v, loc)
         return len(self.newops)
 
-    def finish(self, parent, clt):
-        return ResumeBytecode(self.newops, parent, clt)
+    def finish(self, parent, parent_position, clt):
+        return ResumeBytecode(self.newops, parent, parent_position, clt)
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -259,75 +259,49 @@
         assert self.cpu.tracker.total_compiled_bridges == 1
         return looptoken
 
-    def test_compile_bridge_with_holes(self):
-        i0 = BoxInt()
-        i1 = BoxInt()
-        i2 = BoxInt()
-        i3 = BoxInt()
-        faildescr1 = BasicFailDescr(1)
-        faildescr2 = BasicFailDescr(2)
-        looptoken = JitCellToken()
-        targettoken = TargetToken()
-        operations = [
-            ResOperation(rop.INT_SUB, [i3, ConstInt(42)], i0),
-            ResOperation(rop.LABEL, [i0], None, descr=targettoken),
-            ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
-            ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2),
-            ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1),
-            ResOperation(rop.JUMP, [i1], None, descr=targettoken),
-            ]
-        inputargs = [i3]
-        operations[4].setfailargs([None, i1, None])
-        self.cpu.compile_loop(None, inputargs, operations, looptoken)
-
-        i1b = BoxInt()
-        i3 = BoxInt()
-        bridge = [
-            ResOperation(rop.INT_LE, [i1b, ConstInt(19)], i3),
-            ResOperation(rop.GUARD_TRUE, [i3], None, descr=faildescr2),
-            ResOperation(rop.JUMP, [i1b], None, descr=targettoken),
-        ]
-        bridge[1].setfailargs([i1b])
-
-        self.cpu.compile_bridge(None, faildescr1, [i1b], bridge, looptoken)
-
-        deadframe = self.cpu.execute_token(looptoken, 2)
-        fail = self.cpu.get_latest_descr(deadframe)
-        assert fail.identifier == 2
-        res = self.cpu.get_int_value(deadframe, 0)
-        assert res == 20
-
     def test_compile_big_bridge_out_of_small_loop(self):
+        jitcode = JitCode("name")
+        jitcode.setup(num_regs_i=1, num_regs_r=0, num_regs_f=0)
         i0 = BoxInt()
         faildescr1 = BasicFailDescr(1)
         looptoken = JitCellToken()
         operations = [
+            ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None, descr=jitcode),
+            ResOperation(rop.RESUME_PUT, [i0, ConstInt(0), ConstInt(0)], None),
             ResOperation(rop.GUARD_FALSE, [i0], None, descr=faildescr1),
             ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(2)),
+            ResOperation(rop.LEAVE_FRAME, [], None),
             ]
         inputargs = [i0]
-        operations[0].setfailargs([i0])
         self.cpu.compile_loop(None, inputargs, operations, looptoken)
-
+        jitcode1 = JitCode("name1")
+        jitcode1.setup(num_regs_i=150, num_regs_r=0, num_regs_f=0)
         i1list = [BoxInt() for i in range(150)]
-        bridge = []
+        bridge = [
+            ResOperation(rop.ENTER_FRAME, [ConstInt(13)], None, descr=jitcode1)
+        ]
         iprev = i0
         for i1 in i1list:
             bridge.append(ResOperation(rop.INT_ADD, [iprev, ConstInt(1)], i1))
             iprev = i1
+        for i, i1 in enumerate(i1list):
+            bridge.append(ResOperation(rop.RESUME_PUT, [i1, ConstInt(0), ConstInt(i)], None))
         bridge.append(ResOperation(rop.GUARD_FALSE, [i0], None,
                                    descr=BasicFailDescr(3)))
         bridge.append(ResOperation(rop.FINISH, [], None,
                                    descr=BasicFinalDescr(4)))
-        bridge[-2].setfailargs(i1list)
-
-        self.cpu.compile_bridge(None, faildescr1, [i0], bridge, looptoken)
+        # XXX
+        #bridge[-2].setfailargs(i1list)
+
+        locs = rebuild_locs_from_resumedata(faildescr1)
+        self.cpu.compile_bridge(None, faildescr1, [i0], locs, bridge, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, 1)
         fail = self.cpu.get_latest_descr(deadframe)
+        locs = rebuild_locs_from_resumedata(fail)
         assert fail.identifier == 3
         for i in range(len(i1list)):
-            res = self.cpu.get_int_value(deadframe, i)
+            res = self.cpu.get_int_value(deadframe, locs[i + 1])
             assert res == 2 + i
 
     def test_finish(self):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -476,7 +476,7 @@
         self.update_frame_depth(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE)
         #
         size_excluding_failure_stuff = self.mc.get_relative_pos()
-        self.resume_bytecode = regalloc.resumebuilder.finish(None, looptoken)
+        self.resume_bytecode = regalloc.resumebuilder.finish(None, 0, looptoken)
         self.write_pending_failure_recoveries()
         full_size = self.mc.get_relative_pos()
         #
@@ -538,7 +538,7 @@
         frame_depth_no_fixed_size = self._assemble(regalloc, inputargs, operations)
         codeendpos = self.mc.get_relative_pos()
         self.resume_bytecode = regalloc.resumebuilder.finish(
-            faildescr.rd_resume_bytecode, original_loop_token)
+            faildescr.rd_resume_bytecode, faildescr.rd_bytecode_position, original_loop_token)
         self.write_pending_failure_recoveries()
         fullsize = self.mc.get_relative_pos()
         #
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -324,6 +324,9 @@
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
+            if op.is_guard():
+                self.rm.free_unused_regs()
+                self.xrm.free_unused_regs()
             self.possibly_free_vars_for_op(op)
             self.rm._check_invariants()
             self.xrm._check_invariants()

From noreply at buildbot.pypy.org  Wed Oct 16 16:10:12 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 16:10:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: Tweak tweak tweak: try to ensure
 termination of the marking phase
Message-ID: <20131016141012.678D61C00ED@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67425:bd065e8a99b7
Date: 2013-10-16 16:09 +0200
http://bitbucket.org/pypy/pypy/changeset/bd065e8a99b7/

Log:	Tweak tweak tweak: try to ensure termination of the marking phase

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1040,10 +1040,13 @@
                       "raw_malloc_might_sweep must be empty outside SWEEPING")
 
             if self.gc_state == STATE_MARKING:
-                self._debug_objects_to_trace_dict = \
+                self._debug_objects_to_trace_dict1 = \
                                             self.objects_to_trace.stack2dict()
+                self._debug_objects_to_trace_dict2 = \
+                                       self.more_objects_to_trace.stack2dict()
                 MovingGCBase.debug_check_consistency(self)
-                self._debug_objects_to_trace_dict.delete()
+                self._debug_objects_to_trace_dict2.delete()
+                self._debug_objects_to_trace_dict1.delete()
             else:
                 MovingGCBase.debug_check_consistency(self)
 
@@ -1084,7 +1087,8 @@
         obj = root.address[0]
         if self.header(obj).tid & GCFLAG_VISITED != 0:
             pass    # black -> black
-        elif self._debug_objects_to_trace_dict.contains(obj):
+        elif (self._debug_objects_to_trace_dict1.contains(obj) or
+              self._debug_objects_to_trace_dict2.contains(obj)):
             pass    # black -> gray
         elif self.header(obj).tid & GCFLAG_NO_HEAP_PTRS != 0:
             pass    # black -> white-but-prebuilt-so-dont-care
@@ -1522,7 +1526,7 @@
                 # fully traced very soon.
                 if self.gc_state == STATE_MARKING:
                     self.header(obj).tid &= ~GCFLAG_VISITED
-                    self.objects_to_trace.append(obj)
+                    self.more_objects_to_trace.append(obj)
 
 
     def collect_oldrefs_to_nursery(self):
@@ -1556,7 +1560,7 @@
             # makes sure that we see otherwise-white objects.
             if state_is_marking:
                 self.header(obj).tid &= ~GCFLAG_VISITED
-                self.objects_to_trace.append(obj)
+                self.more_objects_to_trace.append(obj)
             #
             # Trace the 'obj' to replace pointers to nursery with pointers
             # outside the nursery, possibly forcing nursery objects out
@@ -1586,7 +1590,7 @@
             obj = root.address[0]
             if not self.is_in_nursery(obj):
                 if not self.header(obj).tid & GCFLAG_VISITED:
-                    self.objects_to_trace.append(obj)
+                    self.more_objects_to_trace.append(obj)
         #
         self._trace_drag_out(root, None)
 
@@ -1669,7 +1673,7 @@
             # but in the STATE_MARKING phase we still need this bit...
             if self.gc_state == STATE_MARKING:
                 self.header(newobj).tid &= ~GCFLAG_VISITED
-                self.objects_to_trace.append(newobj)
+                self.more_objects_to_trace.append(newobj)
 
     _trace_drag_out._always_inline_ = True
 
@@ -1791,25 +1795,48 @@
             self.objects_to_trace = self.AddressStack()
             self.collect_roots()
             self.gc_state = STATE_MARKING
+            self.more_objects_to_trace = self.AddressStack()
             #END SCANNING
         elif self.gc_state == STATE_MARKING:
             debug_print("number of objects to mark",
-                        self.objects_to_trace.length())
+                        self.objects_to_trace.length(),
+                        "plus",
+                        self.more_objects_to_trace.length())
             estimate = self.gc_increment_step
             estimate_from_nursery = self.nursery_surviving_size * 2
             if estimate_from_nursery > estimate:
                 estimate = estimate_from_nursery
-            self.visit_all_objects_step(intmask(estimate))
+            estimate = intmask(estimate)
+            remaining = self.visit_all_objects_step(estimate)
+            #
+            if remaining >= estimate // 2:
+                if self.more_objects_to_trace.non_empty():
+                    # We consumed less than 1/2 of our step's time, and
+                    # there are more objects added during the marking steps
+                    # of this major collection.  Visit them all now.
+                    # The idea is to ensure termination at the cost of some
+                    # incrementality, in theory.
+                    swap = self.objects_to_trace
+                    self.objects_to_trace = self.more_objects_to_trace
+                    self.more_objects_to_trace = swap
+                    self.visit_all_objects()
 
             # XXX A simplifying assumption that should be checked,
             # finalizers/weak references are rare and short which means that
             # they do not need a seperate state and do not need to be
             # made incremental.
-            if not self.objects_to_trace.non_empty():
+            if (not self.objects_to_trace.non_empty() and
+                not self.more_objects_to_trace.non_empty()):
+                #
                 if self.objects_with_finalizers.non_empty():
                     self.deal_with_objects_with_finalizers()
 
+                ll_assert(not self.objects_to_trace.non_empty(),
+                          "objects_to_trace should be empty")
+                ll_assert(not self.more_objects_to_trace.non_empty(),
+                          "more_objects_to_trace should be empty")
                 self.objects_to_trace.delete()
+                self.more_objects_to_trace.delete()
 
                 #
                 # Weakref support: clear the weak pointers to dying objects
@@ -1933,8 +1960,9 @@
     def start_free_rawmalloc_objects(self):
         ll_assert(not self.raw_malloc_might_sweep.non_empty(),
                   "raw_malloc_might_sweep must be empty")
-        (self.raw_malloc_might_sweep, self.old_rawmalloced_objects) = (
-            self.old_rawmalloced_objects, self.raw_malloc_might_sweep)
+        swap = self.raw_malloc_might_sweep
+        self.raw_malloc_might_sweep = self.old_rawmalloced_objects
+        self.old_rawmalloced_objects = swap
 
     # Returns true when finished processing objects
     def free_unvisited_rawmalloc_objects_step(self, nobjects):
@@ -1981,14 +2009,18 @@
         self.objects_to_trace.append(root.address[0])
 
     def visit_all_objects(self):
-        self.visit_all_objects_step(sys.maxint)
+        while self.objects_to_trace.non_empty():
+            self.visit_all_objects_step(sys.maxint)
 
     def visit_all_objects_step(self, size_to_track):
         # Objects can be added to pending by visit
         pending = self.objects_to_trace
-        while size_to_track > 0 and pending.non_empty():
+        while pending.non_empty():
             obj = pending.pop()
-            size_to_track = self.visit(obj)
+            size_to_track -= self.visit(obj)
+            if size_to_track < 0:
+                return 0
+        return size_to_track
 
     def visit(self, obj):
         #

From noreply at buildbot.pypy.org  Wed Oct 16 16:37:14 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 16:37:14 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: Fix until we start passing tests
Message-ID: <20131016143714.47C6C1C00ED@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67426:3d44b7201a68
Date: 2013-10-16 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/3d44b7201a68/

Log:	Fix until we start passing tests

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -4,7 +4,6 @@
 from rpython.jit.metainterp.history import (INT, REF, FLOAT, JitCellToken,
     ConstInt, BoxInt, AbstractFailDescr)
 from rpython.jit.metainterp.resoperation import ResOperation, rop
-#from rpython.jit.metainterp.resume2 import rebuild_locs_from_resumedata
 from rpython.rlib import rgc
 from rpython.rlib.debug import (debug_start, debug_stop, have_debug_prints,
                                 debug_print)
@@ -121,16 +120,16 @@
         else:
             coeff = 2
         for item, pos in enumerate(loc_positions):
-            if pos < GPR_REGS * WORD:
-                locs[item] = self.cpu.gen_regs[pos // WORD]
-            elif pos < (GPR_REGS + XMM_REGS * coeff) * WORD:
-                pos = (pos // WORD - GPR_REGS) // coeff
+            if pos < GPR_REGS:
+                locs[item] = self.cpu.gen_regs[pos]
+            elif pos < (GPR_REGS + XMM_REGS * coeff):
+                pos = (pos - GPR_REGS) // coeff
                 locs[item] = self.cpu.float_regs[pos]
             else:
-                i = pos // WORD - self.cpu.JITFRAME_FIXED_SIZE
+                i = pos - self.cpu.JITFRAME_FIXED_SIZE
                 assert i >= 0
                 tp = inputargs[input_i].type
-                locs[item] = self.new_stack_loc(i, pos, tp)
+                locs[item] = self.new_stack_loc(i, pos * WORD, tp)
             input_i += 1
         return locs
 
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -339,19 +339,31 @@
         assert isinstance(res, history.AbstractFailDescr)
         return res
 
-    def get_int_value(self, deadframe, pos):
+    def get_int_value(self, deadframe, locs, pos):
         descr = self.gc_ll_descr.getframedescrs(self).arraydescr
         ofs = self.unpack_arraydescr(descr)
+        if locs is None:
+            assert pos == 0
+        else:
+            pos = locs[pos] * WORD
         return self.read_int_at_mem(deadframe, pos + ofs, WORD, 1)
 
-    def get_ref_value(self, deadframe, pos):
+    def get_ref_value(self, deadframe, locs, pos):
         descr = self.gc_ll_descr.getframedescrs(self).arraydescr
         ofs = self.unpack_arraydescr(descr)
+        if locs is None:
+            assert pos == 0
+        else:
+            pos = locs[pos] * WORD
         return self.read_ref_at_mem(deadframe, pos + ofs)
 
-    def get_float_value(self, deadframe, pos):
+    def get_float_value(self, deadframe, locs, pos):
         descr = self.gc_ll_descr.getframedescrs(self).arraydescr
         ofs = self.unpack_arraydescr(descr)
+        if locs is None:
+            assert pos == 0
+        else:
+            pos = locs[pos] * WORD
         return self.read_float_at_mem(deadframe, pos + ofs)
 
     # ____________________ RAW PRIMITIVES ________________________
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -102,22 +102,22 @@
         """Returns the Descr for the last operation executed by the frame."""
         raise NotImplementedError
 
-    def get_int_value(self, deadframe, index):
-        """Returns the value for the index'th argument to the
-        last executed operation (from 'fail_args' if it was a guard,
-        or from 'args' if it was a FINISH).  Returns an int."""
+    def get_int_value(self, deadframe, locations, index):
+        """ Return the value for the index'th argument in the
+        given backend-specific locations. Returns an int.
+        """
         raise NotImplementedError
 
-    def get_float_value(self, deadframe, index):
-        """Returns the value for the index'th argument to the
-        last executed operation (from 'fail_args' if it was a guard,
-        or from 'args' if it was a FINISH).  Returns a FLOATSTORAGE."""
+    def get_float_value(self, deadframe, locations, index):
+        """ Return the value for the index'th argument in the
+        given backend-specific locations. Returns a FLOATSTORAGE.
+        """
         raise NotImplementedError
 
-    def get_ref_value(self, deadframe, index):
-        """Returns the value for the index'th argument to the
-        last executed operation (from 'fail_args' if it was a guard,
-        or from 'args' if it was a FINISH).  Returns a GCREF."""
+    def get_ref_value(self, deadframe, locations, index):
+        """ Return the value for the index'th argument in the
+        given backend-specific locations. Returns a GCREF.
+        """
         raise NotImplementedError
 
     def grab_exc_value(self, deadframe):
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -132,7 +132,7 @@
         self.cpu.compile_loop(None, inputargs, operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
-        res = self.cpu.get_int_value(deadframe, 0)
+        res = self.cpu.get_int_value(deadframe, None, 0)
         assert res == 3
         assert fail.identifier == 1
 
@@ -151,7 +151,7 @@
         deadframe = self.cpu.execute_token(looptoken,
                                            longlong.getfloatstorage(2.8))
         fail = self.cpu.get_latest_descr(deadframe)
-        res = self.cpu.get_float_value(deadframe, 0)
+        res = self.cpu.get_float_value(deadframe, None, 0)
         assert longlong.getrealfloat(res) == 5.1
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 1
@@ -181,7 +181,8 @@
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 2
-        res = self.cpu.get_int_value(deadframe, 0)
+        locs = rebuild_locs_from_resumedata(fail)
+        res = self.cpu.get_int_value(deadframe, locs, 0)
         assert res == 10
 
     def test_backends_dont_keep_loops_alive(self):
@@ -252,7 +253,8 @@
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 2
-        res = self.cpu.get_int_value(deadframe, 0)
+        locs = rebuild_locs_from_resumedata(fail)
+        res = self.cpu.get_int_value(deadframe, locs, 0)
         assert res == 20
 
         assert self.cpu.tracker.total_compiled_loops == 1
@@ -290,18 +292,16 @@
                                    descr=BasicFailDescr(3)))
         bridge.append(ResOperation(rop.FINISH, [], None,
                                    descr=BasicFinalDescr(4)))
-        # XXX
-        #bridge[-2].setfailargs(i1list)
-
-        locs = rebuild_locs_from_resumedata(faildescr1)
-        self.cpu.compile_bridge(None, faildescr1, [i0], locs, bridge, looptoken)
+
+        faillocs = rebuild_locs_from_resumedata(faildescr1)
+        self.cpu.compile_bridge(None, faildescr1, [i0], faillocs, bridge, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, 1)
         fail = self.cpu.get_latest_descr(deadframe)
         locs = rebuild_locs_from_resumedata(fail)
         assert fail.identifier == 3
         for i in range(len(i1list)):
-            res = self.cpu.get_int_value(deadframe, locs[i + 1])
+            res = self.cpu.get_int_value(deadframe, locs, i + 1)
             assert res == 2 + i
 
     def test_finish(self):
@@ -323,7 +323,7 @@
         deadframe = self.cpu.execute_token(looptoken, 99)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail is faildescr
-        res = self.cpu.get_int_value(deadframe, 0)
+        res = self.cpu.get_int_value(deadframe, None, 0)
         assert res == 99
 
         looptoken = JitCellToken()
@@ -334,7 +334,7 @@
         deadframe = self.cpu.execute_token(looptoken)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail is faildescr
-        res = self.cpu.get_int_value(deadframe, 0)
+        res = self.cpu.get_int_value(deadframe, None, 0)
         assert res == 42
 
         looptoken = JitCellToken()
@@ -357,7 +357,7 @@
             deadframe = self.cpu.execute_token(looptoken, value)
             fail = self.cpu.get_latest_descr(deadframe)
             assert fail is faildescr
-            res = self.cpu.get_float_value(deadframe, 0)
+            res = self.cpu.get_float_value(deadframe, None, 0)
             assert longlong.getrealfloat(res) == -61.25
 
             looptoken = JitCellToken()
@@ -368,7 +368,7 @@
             deadframe = self.cpu.execute_token(looptoken)
             fail = self.cpu.get_latest_descr(deadframe)
             assert fail is faildescr
-            res = self.cpu.get_float_value(deadframe, 0)
+            res = self.cpu.get_float_value(deadframe, None, 0)
             assert longlong.getrealfloat(res) == 42.5
 
     def test_execute_operations_in_env(self):

From noreply at buildbot.pypy.org  Wed Oct 16 16:59:37 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 16:59:37 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: a bit of a mess in
 walk_operations, but otherwise pass more tests
Message-ID: <20131016145937.D6F711C3CDC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67427:2bd1ba38dfcd
Date: 2013-10-16 16:58 +0200
http://bitbucket.org/pypy/pypy/changeset/2bd1ba38dfcd/

Log:	a bit of a mess in walk_operations, but otherwise pass more tests

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -649,16 +649,21 @@
             # for tests
             looptoken.compiled_loop_token._ll_initial_locs = locs
 
+    def get_next_op(self, operations, i):
+        while operations[i].is_resume():
+            i += 1
+        return operations[i]
+
     def can_merge_with_next_guard(self, op, i, operations):
         if (op.getopnum() == rop.CALL_MAY_FORCE or
             op.getopnum() == rop.CALL_ASSEMBLER or
             op.getopnum() == rop.CALL_RELEASE_GIL):
-            assert operations[i + 1].getopnum() == rop.GUARD_NOT_FORCED
+            assert self.get_next_op(operations, i + 1).getopnum() == rop.GUARD_NOT_FORCED
             return True
         if not op.is_comparison():
             if op.is_ovf():
-                if (operations[i + 1].getopnum() != rop.GUARD_NO_OVERFLOW and
-                    operations[i + 1].getopnum() != rop.GUARD_OVERFLOW):
+                if (self.get_next_op(operations, i + 1).getopnum() != rop.GUARD_NO_OVERFLOW and
+                    self.get_next_op(operations, i + 1).getopnum() != rop.GUARD_OVERFLOW):
                     not_implemented("int_xxx_ovf not followed by "
                                     "guard_(no)_overflow")
                 return True
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -65,14 +65,16 @@
         deadframe = self.cpu.execute_token(looptoken, *args)
         if self.cpu.get_latest_descr(deadframe) is operations[-1].getdescr():
             self.guard_failed = False
+            locs = None
         else:
             self.guard_failed = True
+            xxxx
         if result_type == 'int':
-            return BoxInt(self.cpu.get_int_value(deadframe, 0))
+            return BoxInt(self.cpu.get_int_value(deadframe, locs, 0))
         elif result_type == 'ref':
-            return BoxPtr(self.cpu.get_ref_value(deadframe, 0))
+            return BoxPtr(self.cpu.get_ref_value(deadframe, locs, 0))
         elif result_type == 'float':
-            return BoxFloat(self.cpu.get_float_value(deadframe, 0))
+            return BoxFloat(self.cpu.get_float_value(deadframe, locs, 0))
         elif result_type == 'void':
             return None
         else:
@@ -378,22 +380,29 @@
         z = BoxInt(579)
         t = BoxInt(455)
         u = BoxInt(0)    # False
+        jitcode = JitCode('jitcode')
+        jitcode.setup(num_regs_i=2, num_regs_r=0, num_regs_f=0)
         looptoken = JitCellToken()
         targettoken = TargetToken()
         operations = [
             ResOperation(rop.LABEL, [y, x], None, descr=targettoken),
+            ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None, descr=jitcode),
             ResOperation(rop.INT_ADD, [x, y], z),
             ResOperation(rop.INT_SUB, [y, ConstInt(1)], t),
             ResOperation(rop.INT_EQ, [t, ConstInt(0)], u),
+            ResOperation(rop.RESUME_PUT, [t, ConstInt(0), ConstInt(0)], None),
+            ResOperation(rop.RESUME_PUT, [z, ConstInt(0), ConstInt(1)], None),
             ResOperation(rop.GUARD_FALSE, [u], None,
                          descr=BasicFailDescr()),
+            ResOperation(rop.LEAVE_FRAME, [], None),
             ResOperation(rop.JUMP, [t, z], None, descr=targettoken),
             ]
-        operations[-2].setfailargs([t, z])
         cpu.compile_loop(None, [x, y], operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 0, 10)
-        assert self.cpu.get_int_value(deadframe, 0) == 0
-        assert self.cpu.get_int_value(deadframe, 1) == 55
+        fail = self.cpu.get_latest_descr(deadframe)
+        locs = rebuild_locs_from_resumedata(fail)
+        assert self.cpu.get_int_value(deadframe, locs, 0) == 0
+        assert self.cpu.get_int_value(deadframe, locs, 1) == 55
 
     def test_int_operations(self):
         from rpython.jit.metainterp.test.test_executor import get_int_tests
@@ -429,24 +438,33 @@
             v2 = BoxInt(testcases[0][1])
             v_res = BoxInt()
             #
+            jitcode = JitCode('jitcode')
+            jitcode.setup(num_regs_i=1, num_regs_f=0, num_regs_r=0)
             if not reversed:
                 ops = [
+                    ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None,
+                                 descr=jitcode),
                     ResOperation(opnum, [v1, v2], v_res),
+                    ResOperation(rop.RESUME_PUT, [v_res, ConstInt(0),
+                                                  ConstInt(0)], None),
                     ResOperation(rop.GUARD_NO_OVERFLOW, [], None,
                                  descr=BasicFailDescr(1)),
+                    ResOperation(rop.LEAVE_FRAME, [], None),
                     ResOperation(rop.FINISH, [v_res], None,
                                  descr=BasicFinalDescr(2)),
                     ]
-                ops[1].setfailargs([])
             else:
-                v_exc = self.cpu.ts.BoxRef()
                 ops = [
+                    ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None,
+                                 descr=jitcode),
                     ResOperation(opnum, [v1, v2], v_res),
+                    ResOperation(rop.RESUME_PUT, [v_res, ConstInt(0),
+                                                  ConstInt(0)], None),
                     ResOperation(rop.GUARD_OVERFLOW, [], None,
                                  descr=BasicFailDescr(1)),
+                    ResOperation(rop.LEAVE_FRAME, [], None),
                     ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(2)),
                     ]
-                ops[1].setfailargs([v_res])
             #
             looptoken = JitCellToken()
             self.cpu.compile_loop(None, [v1, v2], ops, looptoken)
@@ -454,11 +472,13 @@
                 deadframe = self.cpu.execute_token(looptoken, x, y)
                 fail = self.cpu.get_latest_descr(deadframe)
                 if (z == boom) ^ reversed:
+                    locs = rebuild_locs_from_resumedata(fail)
                     assert fail.identifier == 1
                 else:
+                    locs = None
                     assert fail.identifier == 2
                 if z != boom:
-                    assert self.cpu.get_int_value(deadframe, 0) == z
+                    assert self.cpu.get_int_value(deadframe, locs, 0) == z
                 excvalue = self.cpu.grab_exc_value(deadframe)
                 assert not excvalue
 
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -318,8 +318,20 @@
                 self.possibly_free_vars_for_op(op)
                 continue
             if self.can_merge_with_next_guard(op, i, operations):
-                oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
+                # make sure we process all the operations between this and the
+                # next guard before executing it
+                guard_op = self.get_next_op(operations, i + 1)
                 i += 1
+                while operations[i].is_resume():
+                    self.resumebuilder.process(operations[i])
+                    i += 1
+                oplist_with_guard[op.getopnum()](self, op, guard_op)
+                self.rm.position = i
+                self.xrm.position = i
+                self.rm.free_unused_regs()
+                self.xrm.free_unused_regs()
+                i += 1
+                continue
             elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:

From noreply at buildbot.pypy.org  Wed Oct 16 17:19:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 17:19:08 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Link to the binary
Message-ID: <20131016151908.182F91C00ED@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5094:d2353dbdd2d6
Date: 2013-10-16 17:18 +0200
http://bitbucket.org/pypy/extradoc/changeset/d2353dbdd2d6/

Log:	Link to the binary

diff --git a/blog/draft/stm-oct2013.rst b/blog/draft/stm-oct2013.rst
--- a/blog/draft/stm-oct2013.rst
+++ b/blog/draft/stm-oct2013.rst
@@ -78,8 +78,9 @@
 Try it for yourself, but keep in mind that this is still experimental
 with a lot of things yet to come.
 
-You can also download a prebuilt binary from here: **XXX**
-(Linux x64 only for now)
+You can also download a prebuilt binary from here:
+https://bitbucket.org/pypy/pypy/downloads/pypy-oct13-stm.tar.bz2
+(Linux x64 only for now, Ubuntu >= 12.04)
 
 Summary
 -------

From noreply at buildbot.pypy.org  Wed Oct 16 17:29:57 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 17:29:57 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Minor rewordings.
Message-ID: <20131016152957.2B05C1C318B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5095:92b901261eb1
Date: 2013-10-16 17:29 +0200
http://bitbucket.org/pypy/extradoc/changeset/92b901261eb1/

Log:	Minor rewordings.

diff --git a/blog/draft/stm-oct2013.rst b/blog/draft/stm-oct2013.rst
--- a/blog/draft/stm-oct2013.rst
+++ b/blog/draft/stm-oct2013.rst
@@ -21,7 +21,7 @@
 a chance of small errors.
 
 There are still many such performance issues of various complexity left
-to tackle. So stay tuned or contribute :)
+to tackle: we are nowhere near done. So stay tuned or contribute :)
 
 Performance
 -----------
@@ -55,7 +55,7 @@
 
 **Richards** from `PyPy repository on the stmgc-c4
 branch <https://bitbucket.org/pypy/pypy/commits/branch/stmgc-c4>`_:
-Average time per iteration in milliseconds using 8 threads:
+Average time per iteration in milliseconds:
 
 +-------------+----------------------+---------------------+
 | Interpreter | Base time: 1 thread  | 8 threads (speedup) |
@@ -76,11 +76,12 @@
 All this can be found in the `PyPy repository on the stmgc-c4
 branch <https://bitbucket.org/pypy/pypy/commits/branch/stmgc-c4>`_.
 Try it for yourself, but keep in mind that this is still experimental
-with a lot of things yet to come.
+with a lot of things yet to come. Only Linux x64 is supported right
+now, but contributions are welcome.
 
-You can also download a prebuilt binary from here:
+You can download a prebuilt binary from here:
 https://bitbucket.org/pypy/pypy/downloads/pypy-oct13-stm.tar.bz2
-(Linux x64 only for now, Ubuntu >= 12.04)
+(Linux x64 Ubuntu >= 12.04).
 
 Summary
 -------
@@ -89,10 +90,10 @@
 the only of the three interpreters where multithreading gives a large
 improvement in speed.  What they also tell us is that, obviously, the
 result is not good enough *yet:* it still takes longer on a 8-threaded
-PyPy-STM than on a regular single-threaded PyPy-2.1.  As you should know
-by now, we are good at promising speed and delivering it years later.
-It has been two years already since PyPy-STM started, so we're in the
-fast-progressing step right now :-)
+PyPy-STM than on a regular single-threaded PyPy-2.1.  However, as you
+should know by now, we are good at promising speed and delivering it...
+years later ``:-)``  But it has been two years already since PyPy-STM
+started, and things look good now.  Expect major improvements soon.
 
 
 Cheers

From noreply at buildbot.pypy.org  Wed Oct 16 17:39:40 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 17:39:40 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Two more sentences
Message-ID: <20131016153940.DB6451D22DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5096:7369885d6dc1
Date: 2013-10-16 17:39 +0200
http://bitbucket.org/pypy/extradoc/changeset/7369885d6dc1/

Log:	Two more sentences

diff --git a/blog/draft/stm-oct2013.rst b/blog/draft/stm-oct2013.rst
--- a/blog/draft/stm-oct2013.rst
+++ b/blog/draft/stm-oct2013.rst
@@ -92,8 +92,13 @@
 result is not good enough *yet:* it still takes longer on a 8-threaded
 PyPy-STM than on a regular single-threaded PyPy-2.1.  However, as you
 should know by now, we are good at promising speed and delivering it...
-years later ``:-)``  But it has been two years already since PyPy-STM
-started, and things look good now.  Expect major improvements soon.
+years later ``:-)``
+
+But it has been two years already since PyPy-STM started, and this is
+our first preview of the JIT integration.  Expect major improvements
+soon: with STM, the JIT generates code that is completely suboptimal in
+many cases (barriers, allocation, and more).  Once we improve this, the
+performance of the STM-JITted code should come much closer to PyPy 2.1.
 
 
 Cheers

From noreply at buildbot.pypy.org  Wed Oct 16 17:46:09 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 17:46:09 +0200 (CEST)
Subject: [pypy-commit] buildbot default: The issue was fixed in bd065e8a99b7
Message-ID: <20131016154609.E868F1D22DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r869:79aa8d6bb6a3
Date: 2013-10-16 17:45 +0200
http://bitbucket.org/pypy/buildbot/changeset/79aa8d6bb6a3/

Log:	The issue was fixed in bd065e8a99b7

diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -215,8 +215,7 @@
             JITFREEBSD764,             # on headless
             JITFREEBSD864,             # on ananke
             JITFREEBSD964,             # on exarkun's freebsd
-            #JITMACOSX64,              # on xerxes
-            #^^^ status: "py.test -A" eats 20GB of memory apparently :-(
+            JITMACOSX64,               # on xerxes
             # buildbot selftest
             PYPYBUILDBOT               # on cobra
             ], branch='default', hour=0, minute=0),

From noreply at buildbot.pypy.org  Wed Oct 16 17:51:09 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 16 Oct 2013 17:51:09 +0200 (CEST)
Subject: [pypy-commit] stmgc default: make thread_descriptor accessible to
	pypy
Message-ID: <20131016155109.EAFF01D22DC@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r537:3acc863a00a8
Date: 2013-10-16 17:50 +0200
http://bitbucket.org/pypy/stmgc/changeset/3acc863a00a8/

Log:	make thread_descriptor accessible to pypy

diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -167,7 +167,7 @@
   gcptr **shadowstack_end_ref;
   gcptr *thread_local_obj_ref;
   gcptr old_thread_local_obj;
-
+  /* sync with pypy stmgc: */
   NURSERY_FIELDS_DECL
 
   long atomic;   /* 0 = not atomic, > 0 atomic */
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -201,6 +201,7 @@
 #define stm_pop_root()      (*--stm_shadowstack)
 
 extern __thread revision_t stm_private_rev_num;
+extern __thread struct tx_descriptor *thread_descriptor; /* XXX: stm_ prefix */
 gcptr stm_DirectReadBarrier(gcptr);
 gcptr stm_WriteBarrier(gcptr);
 gcptr stm_RepeatReadBarrier(gcptr);

From noreply at buildbot.pypy.org  Wed Oct 16 17:54:45 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 16 Oct 2013 17:54:45 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: remove unnecessary write barrier on
	jitframe
Message-ID: <20131016155445.27DD51D22DC@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67428:ac7daebd5480
Date: 2013-10-15 17:43 +0200
http://bitbucket.org/pypy/pypy/changeset/ac7daebd5480/

Log:	remove unnecessary write barrier on jitframe

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -965,19 +965,9 @@
         return rst
 
     def _call_header_shadowstack(self, gcrootmap):
-        # do a write-barrier on ebp / frame for stm
-        # XXX: may not be necessary if we are sure that we only get
-        #      freshly allocated frames or already write-ready frames
-        #      from the caller...
-        gc_ll_descr = self.cpu.gc_ll_descr
-        gcrootmap = gc_ll_descr.gcrootmap
-        if gcrootmap and gcrootmap.is_stm:
-            if not hasattr(gc_ll_descr, 'P2Wdescr'):
-                raise Exception("unreachable code")
-            wbdescr = gc_ll_descr.P2Wdescr
-            self._stm_barrier_fastpath(self.mc, wbdescr, [ebp], is_frame=True)
-
         # put the frame in ebp on the shadowstack for the GC to find
+        # (ebp is a writeable object and does not need a write-barrier
+        # again (ensured by the code calling the loop))
         rst = self._load_shadowstack_top_in_ebx(self.mc, gcrootmap)
         self.mc.MOV_mr((ebx.value, 0), ebp.value)      # MOV [ebx], ebp
         self.mc.ADD_ri(ebx.value, WORD)

From noreply at buildbot.pypy.org  Wed Oct 16 17:54:46 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 16 Oct 2013 17:54:46 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: import stmgc for nursery optimizations
	in JIT
Message-ID: <20131016155446.4C0661D22DC@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67429:3a991051a045
Date: 2013-10-16 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/3a991051a045/

Log:	import stmgc for nursery optimizations in JIT

diff --git a/rpython/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c
--- a/rpython/translator/stm/src_stm/et.c
+++ b/rpython/translator/stm/src_stm/et.c
@@ -24,7 +24,7 @@
         i++;
     }
     cur += sprintf(cur, "tid=%ld", stm_get_tid(obj));
-    cur += sprintf(cur, " : rev=%lx : orig=%lx", 
+    cur += sprintf(cur, " : rev=0x%lx : orig=0x%lx", 
                    (long)obj->h_revision, (long)obj->h_original);
     return tmp_buf;
 }
diff --git a/rpython/translator/stm/src_stm/et.h b/rpython/translator/stm/src_stm/et.h
--- a/rpython/translator/stm/src_stm/et.h
+++ b/rpython/translator/stm/src_stm/et.h
@@ -168,7 +168,7 @@
   gcptr **shadowstack_end_ref;
   gcptr *thread_local_obj_ref;
   gcptr old_thread_local_obj;
-
+  /* sync with pypy stmgc: */
   NURSERY_FIELDS_DECL
 
   long atomic;   /* 0 = not atomic, > 0 atomic */
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-9149deb7e746
+3acc863a00a8
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -202,6 +202,7 @@
 #define stm_pop_root()      (*--stm_shadowstack)
 
 extern __thread revision_t stm_private_rev_num;
+extern __thread struct tx_descriptor *thread_descriptor; /* XXX: stm_ prefix */
 gcptr stm_DirectReadBarrier(gcptr);
 gcptr stm_WriteBarrier(gcptr);
 gcptr stm_RepeatReadBarrier(gcptr);
diff --git a/rpython/translator/stm/src_stm/stmsync.c b/rpython/translator/stm/src_stm/stmsync.c
--- a/rpython/translator/stm/src_stm/stmsync.c
+++ b/rpython/translator/stm/src_stm/stmsync.c
@@ -223,14 +223,14 @@
     }
     else {
         CommitTransaction();
-        if (d->active != 2) {
-            unsigned long limit = d->reads_size_limit_nonatomic;
-            if (limit != 0 && limit < (stm_regular_length_limit >> 1))
-                limit = (limit << 1) | 1;
-            else
-                limit = stm_regular_length_limit;
-            d->reads_size_limit_nonatomic = limit;
-        }
+
+        unsigned long limit = d->reads_size_limit_nonatomic;
+        if (limit != 0 && limit < (stm_regular_length_limit >> 1))
+            limit = (limit << 1) | 1;
+        else
+            limit = stm_regular_length_limit;
+        d->reads_size_limit_nonatomic = limit;
+
         stm_begin_transaction(buf, longjmp_callback);
     }
 }

From noreply at buildbot.pypy.org  Wed Oct 16 17:54:47 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 16 Oct 2013 17:54:47 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: start implementing fastpath for
 nursery allocations (WIP)
Message-ID: <20131016155447.AAE501D22DC@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67430:53c3d84d1993
Date: 2013-10-16 17:53 +0200
http://bitbucket.org/pypy/pypy/changeset/53c3d84d1993/

Log:	start implementing fastpath for nursery allocations (WIP) copy over
	rewrite tests for stm (need fixing)

diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -511,11 +511,7 @@
             self._make_layoutbuilder()
             self._make_gcrootmap()
             self._setup_gcclass()
-            if not self.stm:
-                # XXX: not needed with stm/shadowstack??
-                self._setup_tid()
-            else:
-                self.fielddescr_tid = None
+            self._setup_tid()
         self._setup_write_barrier()
         self._setup_str()
         self._make_functions(really_not_translated)
@@ -534,10 +530,8 @@
     def _initialize_for_tests(self):
         self.layoutbuilder = None
         self.fielddescr_tid = AbstractDescr()
-        if self.stm:
-            self.max_size_of_young_obj = None
-        else:
-            self.max_size_of_young_obj = 1000
+        self.fielddescr_rev = AbstractDescr()
+        self.max_size_of_young_obj = 1000
         self.GCClass = None
         self.gcheaderbuilder = None
         self.HDRPTR = None
@@ -572,7 +566,15 @@
         assert self.GCClass.inline_simple_malloc_varsize
 
     def _setup_tid(self):
-        self.fielddescr_tid = get_field_descr(self, self.GCClass.HDR, 'tid')
+        if not self.stm:
+            self.fielddescr_tid = get_field_descr(self, self.GCClass.HDR, 'tid')
+            self.fielddescr_rev = None
+        else:
+            self.fielddescr_tid = get_field_descr(self, self.GCClass.GCHDR,
+                                                  'h_tid')
+            self.fielddescr_rev = get_field_descr(self, self.GCClass.GCHDR,
+                                                  'h_revision')
+                        
         frame_tid = self.layoutbuilder.get_type_id(jitframe.JITFRAME)
         self.translator._jit2gc['frame_tid'] = frame_tid
 
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -52,6 +52,7 @@
         # barriers.  We do this on each "basic block" of operations, which in
         # this case means between CALLs or unknown-size mallocs.
         #
+        # SYNC with stmrewrite.py!
         for op in operations:
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -88,10 +88,11 @@
                 # it immediately
                 if (op.getopnum() == rop.GUARD_NOT_FORCED
                     and insert_transaction_break):
-                    # insert transaction_break after GUARD after call
+                    # insert transaction_break after GUARD after calls
                     self.newops.append(
                         ResOperation(rop.STM_TRANSACTION_BREAK, [], None))
                     insert_transaction_break = False
+                    self.emitting_an_operation_that_can_collect()
                 else:
                     assert insert_transaction_break is False
 
@@ -118,6 +119,7 @@
                 continue
             # ----------  calls  ----------
             if op.is_call():
+                self.emitting_an_operation_that_can_collect()
                 if (op.getopnum() == rop.CALL_MAY_FORCE or
                     op.getopnum() == rop.CALL_ASSEMBLER or
                     op.getopnum() == rop.CALL_RELEASE_GIL):
@@ -142,7 +144,6 @@
                         self.fallback_inevitable(op)
                     else:
                         self.newops.append(op)
-                self.known_category.clear()
                 continue
             # ----------  copystrcontent  ----------
             if op.getopnum() in (rop.COPYSTRCONTENT,
@@ -155,7 +156,8 @@
                     continue
             # ----------  labels  ----------
             if op.getopnum() == rop.LABEL:
-                self.known_category.clear()
+                self.emitting_an_operation_that_can_collect()
+                self.known_lengths.clear()
                 self.always_inevitable = False
                 self.newops.append(op)
                 continue
@@ -163,6 +165,7 @@
             if op.getopnum() == rop.JUMP:
                 self.newops.append(
                     ResOperation(rop.STM_TRANSACTION_BREAK, [], None))
+                # self.emitting_an_operation_that_can_collect()
                 self.newops.append(op)
                 continue
             # ----------  finish, other ignored ops  ----------
@@ -185,6 +188,10 @@
         assert not insert_transaction_break
         return self.newops
 
+    def emitting_an_operation_that_can_collect(self):
+        GcRewriterAssembler.emitting_an_operation_that_can_collect(self)
+        self.known_category.clear()
+
     def write_to_read_categories(self):
         for v, c in self.known_category.items():
             if c == 'W':
@@ -197,13 +204,14 @@
             if c == 'R':
                 self.known_category[v] = 'P'
 
-##    def gen_malloc_nursery_varsize_frame(self, sizebox, v_result, tid):
-##        """ For now don't generate CALL_MALLOC_NURSERY_VARSIZE_FRAME
-##        """
-##        addr = self.gc_ll_descr.get_malloc_fn_addr('malloc_big_fixedsize')
-##        args = [ConstInt(addr), sizebox, ConstInt(tid)]
-##        descr = self.gc_ll_descr.malloc_big_fixedsize_descr
-##        self._gen_call_malloc_gc(args, v_result, descr)
+    def gen_initialize_tid(self, v_newgcobj, tid):
+        GcRewriterAssembler.gen_initialize_tid(self, v_newgcobj, tid)
+        if self.gc_ll_descr.fielddescr_rev is not None:
+            op = ResOperation(rop.STM_SET_REVISION_GC, [v_newgcobj,], None,
+                              descr=self.gc_ll_descr.fielddescr_rev)
+            self.newops.append(op)
+            
+
                 
     def gen_write_barrier(self, v):
         raise NotImplementedError
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -63,6 +63,7 @@
         register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
+        revdescr = self.gc_ll_descr.fielddescr_rev
         wbdescr = self.gc_ll_descr.write_barrier_descr
         WORD = globals()['WORD']
         #
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -744,7 +744,6 @@
                                     fakeextrainfo())
         for op, guarded in [
                 ("call(123, descr=calldescr2)", False),
-                ("call_assembler(123, descr=casmdescr)", True),
                 ("call_may_force(123, descr=calldescr2)", True),
                 ("call_loopinvariant(123, descr=calldescr2)", False),
                 ]:
@@ -770,6 +769,27 @@
                 jump(p1)
             """ % (op, guard, tr_break), calldescr2=calldescr2)
 
+    def test_call_assembler(self):
+        self.check_rewrite("""
+        [i0, f0]
+        i2 = call_assembler(i0, f0, descr=casmdescr)
+        guard_not_forced()[] 
+        """, """
+        [i0, f0]
+        i1 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_size)
+        p1 = call_malloc_nursery_varsize_frame(i1)
+        setfield_gc(p1, 0, descr=tiddescr)
+        stm_set_revision_gc(p1, descr=revdescr)
+        i2 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth)
+        setfield_gc(p1, i2, descr=framelendescr)
+        setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info)
+        setarrayitem_gc(p1, 0, i0, descr=signedframedescr)
+        setarrayitem_gc(p1, 1, f0, descr=floatframedescr)
+        i3 = call_assembler(p1, descr=casmdescr)
+        guard_not_forced() []
+        stm_transaction_break()
+        """)
+
     def test_ptr_eq_null(self):
         self.check_rewrite("""
             [p1, p2]
@@ -833,3 +853,273 @@
     def test_ptr_eq_other_direct_cases(self):
         py.test.skip("can also keep ptr_eq if both args are L or W, "
                      "or if one arg is freshly malloced")
+
+    # ----------- tests copied from rewrite.py -------------
+    def test_rewrite_assembler_new_to_malloc(self):
+        self.check_rewrite("""
+            [p1]
+            p0 = new(descr=sdescr)
+        """, """
+            [p1]
+            p0 = call_malloc_nursery(%(sdescr.size)d)
+            setfield_gc(p0, 1234, descr=tiddescr)
+            stm_set_revision_gc(p0, descr=revdescr)
+        """)
+
+    def test_rewrite_assembler_new3_to_malloc(self):
+        self.check_rewrite("""
+            []
+            p0 = new(descr=sdescr)
+            p1 = new(descr=tdescr)
+            p2 = new(descr=sdescr)
+            jump()
+        """, """
+            []
+            p0 = call_malloc_nursery(   \
+                               %(sdescr.size + tdescr.size + sdescr.size)d)
+            setfield_gc(p0, 1234, descr=tiddescr)
+            p1 = int_add(p0, %(sdescr.size)d)
+            setfield_gc(p1, 5678, descr=tiddescr)
+            p2 = int_add(p1, %(tdescr.size)d)
+            setfield_gc(p2, 1234, descr=tiddescr)
+            jump()
+        """)
+
+    def test_rewrite_assembler_new_array_fixed_to_malloc(self):
+        self.check_rewrite("""
+            []
+            p0 = new_array(10, descr=adescr)
+            jump()
+        """, """
+            []
+            p0 = call_malloc_nursery(    \
+                                %(adescr.basesize + 10 * adescr.itemsize)d)
+            setfield_gc(p0, 4321, descr=tiddescr)
+            setfield_gc(p0, 10, descr=alendescr)
+            jump()
+        """)
+
+    def test_rewrite_assembler_new_and_new_array_fixed_to_malloc(self):
+        self.check_rewrite("""
+            []
+            p0 = new(descr=sdescr)
+            p1 = new_array(10, descr=adescr)
+            jump()
+        """, """
+            []
+            p0 = call_malloc_nursery(                                  \
+                                %(sdescr.size +                        \
+                                  adescr.basesize + 10 * adescr.itemsize)d)
+            setfield_gc(p0, 1234, descr=tiddescr)
+            p1 = int_add(p0, %(sdescr.size)d)
+            setfield_gc(p1, 4321, descr=tiddescr)
+            setfield_gc(p1, 10, descr=alendescr)
+            jump()
+        """)
+
+    def test_rewrite_assembler_round_up(self):
+        self.check_rewrite("""
+            []
+            p0 = new_array(6, descr=bdescr)
+            jump()
+        """, """
+            []
+            p0 = call_malloc_nursery(%(bdescr.basesize + 8)d)
+            setfield_gc(p0, 8765, descr=tiddescr)
+            setfield_gc(p0, 6, descr=blendescr)
+            jump()
+        """)
+
+    def test_rewrite_assembler_round_up_always(self):
+        self.check_rewrite("""
+            []
+            p0 = new_array(5, descr=bdescr)
+            p1 = new_array(5, descr=bdescr)
+            p2 = new_array(5, descr=bdescr)
+            p3 = new_array(5, descr=bdescr)
+            jump()
+        """, """
+            []
+            p0 = call_malloc_nursery(%(4 * (bdescr.basesize + 8))d)
+            setfield_gc(p0, 8765, descr=tiddescr)
+            setfield_gc(p0, 5, descr=blendescr)
+            p1 = int_add(p0, %(bdescr.basesize + 8)d)
+            setfield_gc(p1, 8765, descr=tiddescr)
+            setfield_gc(p1, 5, descr=blendescr)
+            p2 = int_add(p1, %(bdescr.basesize + 8)d)
+            setfield_gc(p2, 8765, descr=tiddescr)
+            setfield_gc(p2, 5, descr=blendescr)
+            p3 = int_add(p2, %(bdescr.basesize + 8)d)
+            setfield_gc(p3, 8765, descr=tiddescr)
+            setfield_gc(p3, 5, descr=blendescr)
+            jump()
+        """)
+
+    def test_rewrite_assembler_minimal_size(self):
+        self.check_rewrite("""
+            []
+            p0 = new(descr=edescr)
+            p1 = new(descr=edescr)
+            jump()
+        """, """
+            []
+            p0 = call_malloc_nursery(%(4*WORD)d)
+            setfield_gc(p0, 9000, descr=tiddescr)
+            p1 = int_add(p0, %(2*WORD)d)
+            setfield_gc(p1, 9000, descr=tiddescr)
+            jump()
+        """)
+
+    def test_rewrite_assembler_variable_size(self):
+        self.check_rewrite("""
+            [i0]
+            p0 = new_array(i0, descr=bdescr)
+            jump(i0)
+        """, """
+            [i0]
+            p0 = call_malloc_nursery_varsize(0, 1, i0, descr=bdescr)
+            setfield_gc(p0, i0, descr=blendescr)
+            jump(i0)
+        """)
+
+    def test_rewrite_new_string(self):
+        self.check_rewrite("""
+        [i0]
+        p0 = newstr(i0)
+        jump(i0)
+        """, """
+        [i0]
+        p0 = call_malloc_nursery_varsize(1, 1, i0, descr=strdescr)
+        setfield_gc(p0, i0, descr=strlendescr)
+        jump(i0)
+        """)
+
+    def test_rewrite_assembler_nonstandard_array(self):
+        # a non-standard array is a bit hard to get; e.g. GcArray(Float)
+        # is like that on Win32, but not on Linux.  Build one manually...
+        NONSTD = lltype.GcArray(lltype.Float)
+        nonstd_descr = get_array_descr(self.gc_ll_descr, NONSTD)
+        nonstd_descr.tid = 6464
+        nonstd_descr.basesize = 64      # <= hacked
+        nonstd_descr.itemsize = 8
+        nonstd_descr_gcref = 123
+        self.check_rewrite("""
+            [i0]
+            p0 = new_array(i0, descr=nonstd_descr)
+            jump(i0)
+        """, """
+            [i0]
+            p0 = call_malloc_gc(ConstClass(malloc_array_nonstandard), \
+                                64, 8,                                \
+                                %(nonstd_descr.lendescr.offset)d,     \
+                                6464, i0,                             \
+                                descr=malloc_array_nonstandard_descr)
+            jump(i0)
+        """, nonstd_descr=nonstd_descr)
+
+    def test_rewrite_assembler_maximal_size_1(self):
+        self.gc_ll_descr.max_size_of_young_obj = 100
+        self.check_rewrite("""
+            []
+            p0 = new_array(103, descr=bdescr)
+            jump()
+        """, """
+            []
+            p0 = call_malloc_gc(ConstClass(malloc_array), 1,  \
+                                %(bdescr.tid)d, 103,          \
+                                descr=malloc_array_descr)
+            jump()
+        """)
+
+    def test_rewrite_assembler_maximal_size_2(self):
+        self.gc_ll_descr.max_size_of_young_obj = 300
+        self.check_rewrite("""
+            []
+            p0 = new_array(101, descr=bdescr)
+            p1 = new_array(102, descr=bdescr)  # two new_arrays can be combined
+            p2 = new_array(103, descr=bdescr)  # but not all three
+            jump()
+        """, """
+            []
+            p0 = call_malloc_nursery(    \
+                              %(2 * (bdescr.basesize + 104))d)
+            setfield_gc(p0, 8765, descr=tiddescr)
+            setfield_gc(p0, 101, descr=blendescr)
+            p1 = int_add(p0, %(bdescr.basesize + 104)d)
+            setfield_gc(p1, 8765, descr=tiddescr)
+            setfield_gc(p1, 102, descr=blendescr)
+            p2 = call_malloc_nursery(    \
+                              %(bdescr.basesize + 104)d)
+            setfield_gc(p2, 8765, descr=tiddescr)
+            setfield_gc(p2, 103, descr=blendescr)
+            jump()
+        """)
+
+    def test_rewrite_assembler_huge_size(self):
+        # "huge" is defined as "larger than 0xffffff bytes, or 16MB"
+        self.check_rewrite("""
+            []
+            p0 = new_array(20000000, descr=bdescr)
+            jump()
+        """, """
+            []
+            p0 = call_malloc_gc(ConstClass(malloc_array), 1, \
+                                %(bdescr.tid)d, 20000000,    \
+                                descr=malloc_array_descr)
+            jump()
+        """)
+
+    def test_new_with_vtable(self):
+        self.check_rewrite("""
+            []
+            p0 = new_with_vtable(ConstClass(o_vtable))
+            jump()
+        """, """
+            [p1]
+            p0 = call_malloc_nursery(104)      # rounded up
+            setfield_gc(p0, 9315, descr=tiddescr)
+            setfield_gc(p0, ConstClass(o_vtable), descr=vtable_descr)
+            jump()
+        """)
+
+    def test_new_with_vtable_too_big(self):
+        self.gc_ll_descr.max_size_of_young_obj = 100
+        self.check_rewrite("""
+            []
+            p0 = new_with_vtable(ConstClass(o_vtable))
+            jump()
+        """, """
+            [p1]
+            p0 = call_malloc_gc(ConstClass(malloc_big_fixedsize), 104, 9315, \
+                                descr=malloc_big_fixedsize_descr)
+            setfield_gc(p0, ConstClass(o_vtable), descr=vtable_descr)
+            jump()
+        """)
+
+    def test_rewrite_assembler_newstr_newunicode(self):
+        self.check_rewrite("""
+            [i2]
+            p0 = newstr(14)
+            p1 = newunicode(10)
+            p2 = newunicode(i2)
+            p3 = newstr(i2)
+            jump()
+        """, """
+            [i2]
+            p0 = call_malloc_nursery(                                \
+                      %(strdescr.basesize + 16 * strdescr.itemsize + \
+                        unicodedescr.basesize + 10 * unicodedescr.itemsize)d)
+            setfield_gc(p0, %(strdescr.tid)d, descr=tiddescr)
+            setfield_gc(p0, 14, descr=strlendescr)
+            p1 = int_add(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d)
+            setfield_gc(p1, %(unicodedescr.tid)d, descr=tiddescr)
+            setfield_gc(p1, 10, descr=unicodelendescr)
+            p2 = call_malloc_nursery_varsize(2, 4, i2, \
+                                descr=unicodedescr)
+            setfield_gc(p2, i2, descr=unicodelendescr)
+            p3 = call_malloc_nursery_varsize(1, 1, i2, \
+                                descr=strdescr)
+            setfield_gc(p3, i2, descr=strlendescr)
+            jump()
+        """)
+
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -254,11 +254,18 @@
         mc.J_il(rx86.Conditions['Z'], 0xfffff) # patched later
         jz_location = mc.get_relative_pos()
         #
-        nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr()
         self._reload_frame_if_necessary(mc, align_stack=True)
         self.set_extra_stack_depth(mc, 0)
         self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats)
-        mc.MOV(edi, heap(nursery_free_adr))   # load this in EDI
+        if self.cpu.gc_ll_descr.stm:
+            # load nursery_current into EDI
+            self._load_stm_thread_descriptor(mc, X86_64_SCRATCH_REG)
+            mc.MOV_rm(edi.value, 
+                      (X86_64_SCRATCH_REG.value, 
+                       StmGC.TD_NURSERY_CURRENT))
+        else:
+            nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr()
+            mc.MOV(edi, heap(nursery_free_adr))   # load this in EDI
         # clear the gc pattern
         mc.MOV_bi(ofs, 0)
         mc.RET()
@@ -2748,6 +2755,175 @@
         # XXX if the next operation is a GUARD_NO_EXCEPTION, we should
         # somehow jump over it too in the fast path
 
+    def _load_stm_thread_descriptor(self, mc, loc):
+        assert self.cpu.gc_ll_descr.stm
+        assert isinstance(loc, RegLoc)
+        
+        td = self._get_stm_tl(rstm.get_thread_descriptor_adr())
+        self._tl_segment_if_stm(mc)
+        mc.MOV(loc, heap(td))
+        mc.MOV_rm(loc.value, (loc.value, 0))
+
+    def _cond_allocate_in_nursery_or_slowpath(self, mc, gcmap):
+        # needed for slowpath:
+        # eax = nursery_current
+        # edi = nursery_current + size
+        # needed here:
+        # X86_64_SCRATCH_REG = thread_descriptor
+        #
+        # cmp nursery_current+size > nursery_nextlimit
+        mc.CMP_rm(edi.value, (X86_64_SCRATCH_REG.value, 
+                              StmGC.TD_NURSERY_NEXTLIMIT))
+        mc.J_il8(rx86.Conditions['NA'], 0) # patched later
+        jmp_adr = mc.get_relative_pos()
+        #
+        # == SLOWPATH ==
+        # save the gcmap
+        self.push_gcmap(mc, gcmap, mov=True)
+        mc.CALL(imm(self.malloc_slowpath))
+        mc.JMP_l8(0)
+        jmp2_adr = mc.get_relative_pos()
+        #
+        # == FASTPATH ==
+        offset = mc.get_relative_pos() - jmp_adr
+        assert 0 < offset <= 127
+        mc.overwrite(jmp_adr-1, chr(offset))
+        #
+        # thread_descriptor->nursery_current = nursery_current+size
+        mc.MOV_mr((X86_64_SCRATCH_REG.value,
+                   StmGC.TD_NURSERY_CURRENT),
+                   edi.value)
+        #
+        # END
+        offset = mc.get_relative_pos() - jmp2_adr
+        assert 0 < offset <= 127
+        mc.overwrite(jmp2_adr-1, chr(offset))
+        
+    def malloc_cond_stm(self, size, gcmap):
+        assert self.cpu.gc_ll_descr.stm
+        assert size & (WORD-1) == 0     # must be correctly aligned
+        mc = self.mc
+        # load nursery_current and nursery_nextlimit
+        self._load_stm_thread_descriptor(mc, X86_64_SCRATCH_REG)
+        mc.MOV_rm(eax.value, 
+                  (X86_64_SCRATCH_REG.value,
+                   StmGC.TD_NURSERY_CURRENT))
+        mc.LEA_rm(edi.value, (eax.value, size))
+        #
+        # eax=nursery_current, edi=nursery_current+size
+        self._cond_allocate_in_nursery_or_slowpath(mc, gcmap)
+
+    def malloc_cond_varsize_frame_stm(self, sizeloc, gcmap):
+        assert self.cpu.gc_ll_descr.stm
+        mc = self.mc
+        self._load_stm_thread_descriptor(mc, X86_64_SCRATCH_REG)
+        if sizeloc is eax:
+            self.mc.MOV(edi, sizeloc)
+            sizeloc = edi
+        self.mc.MOV_rm(eax.value, (X86_64_SCRATCH_REG.value, 
+                                   StmGC.TD_NURSERY_CURRENT))
+        if sizeloc is edi:
+            self.mc.ADD_rr(edi.value, eax.value)
+        else:
+            self.mc.LEA_ra(edi.value, (eax.value, sizeloc.value, 0, 0))
+        #
+        # eax=nursery_current, edi=nursery_current+size
+        self._cond_allocate_in_nursery_or_slowpath(mc, gcmap)
+
+    def malloc_cond_varsize_stm(self, kind, lengthloc, itemsize,
+                                maxlength, gcmap, arraydescr):
+        assert self.cpu.gc_ll_descr.stm
+        from rpython.jit.backend.llsupport.descr import ArrayDescr
+        assert isinstance(arraydescr, ArrayDescr)
+
+        mc = self.mc
+        # lengthloc is the length of the array, which we must not modify!
+        assert lengthloc is not eax and lengthloc is not edi
+        if isinstance(lengthloc, RegLoc):
+            varsizeloc = lengthloc
+        else:
+            mc.MOV(edi, lengthloc)
+            varsizeloc = edi
+
+        mc.CMP(varsizeloc, imm(maxlength))
+        mc.J_il8(rx86.Conditions['A'], 0) # patched later
+        jmp_adr0 = mc.get_relative_pos()
+
+        self._load_stm_thread_descriptor(mc, X86_64_SCRATCH_REG)
+        mc.MOV_rm(eax.value, 
+                  (X86_64_SCRATCH_REG.value, 
+                   StmGC.TD_NURSERY_CURRENT))
+
+        if valid_addressing_size(itemsize):
+            shift = get_scale(itemsize)
+        else:
+            shift = self._imul_const_scaled(mc, edi.value,
+                                            varsizeloc.value, itemsize)
+            varsizeloc = edi
+        # now varsizeloc is a register != eax.  The size of
+        # the variable part of the array is (varsizeloc << shift)
+        assert arraydescr.basesize >= self.gc_minimal_size_in_nursery
+        constsize = arraydescr.basesize + self.gc_size_of_header
+        force_realignment = (itemsize % WORD) != 0
+        if force_realignment:
+            constsize += WORD - 1
+        mc.LEA_ra(edi.value, (eax.value, varsizeloc.value, shift,
+                                   constsize))
+        if force_realignment:
+            mc.AND_ri(edi.value, ~(WORD - 1))
+        # now edi contains the total size in bytes, rounded up to a multiple
+        # of WORD, plus nursery_free_adr
+        mc.CMP_rm(edi.value, (X86_64_SCRATCH_REG.value, 
+                              StmGC.TD_NURSERY_NEXTLIMIT))
+        mc.J_il8(rx86.Conditions['NA'], 0) # patched later
+        jmp_adr1 = mc.get_relative_pos()
+        #
+        # == SLOWPATH ==
+        offset = mc.get_relative_pos() - jmp_adr0
+        assert 0 < offset <= 127
+        mc.overwrite(jmp_adr0-1, chr(offset))
+        # save the gcmap
+        self.push_gcmap(mc, gcmap, mov=True)   # mov into RawEspLoc(0)
+        if kind == rewrite.FLAG_ARRAY:
+            mc.MOV_si(WORD, itemsize)
+            mc.MOV(edi, lengthloc)
+            mc.MOV_ri(eax.value, arraydescr.tid)
+            addr = self.malloc_slowpath_varsize
+        else:
+            if kind == rewrite.FLAG_STR:
+                addr = self.malloc_slowpath_str
+            else:
+                assert kind == rewrite.FLAG_UNICODE
+                addr = self.malloc_slowpath_unicode
+            mc.MOV(edi, lengthloc)
+        mc.CALL(imm(addr))
+        mc.JMP_l8(0)      # jump to done, patched later
+        jmp_location = mc.get_relative_pos()
+        #
+        # == FASTPATH ==
+        offset = mc.get_relative_pos() - jmp_adr1
+        assert 0 < offset <= 127
+        mc.overwrite(jmp_adr1-1, chr(offset))
+        #
+        # set thread_descriptor->nursery_current
+        mc.MOV_mr((X86_64_SCRATCH_REG.value,
+                   StmGC.TD_NURSERY_CURRENT),
+                   edi.value)
+        #
+        # write down the tid
+        mc.MOV(mem(eax, 0), imm(arraydescr.tid))
+        # also set private_rev_num:
+        rn = self._get_stm_private_rev_num_addr()
+        self._tl_segment_if_stm(mc)
+        mc.MOV_rj(X86_64_SCRATCH_REG.value, rn)
+        mc.MOV(mem(eax, StmGC.H_REVISION), X86_64_SCRATCH_REG)
+        #
+        # == END ==
+        offset = mc.get_relative_pos() - jmp_location
+        assert 0 < offset <= 127
+        mc.overwrite(jmp_location - 1, chr(offset))
+
+    
     def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap):
         assert not self.cpu.gc_ll_descr.stm
         assert size & (WORD-1) == 0     # must be correctly aligned
@@ -2764,6 +2940,7 @@
         self.mc.overwrite(jmp_adr-1, chr(offset))
         self.mc.MOV(heap(nursery_free_adr), edi)
 
+        
     def malloc_cond_varsize_frame(self, nursery_free_adr, nursery_top_adr,
                                   sizeloc, gcmap):
         assert not self.cpu.gc_ll_descr.stm
@@ -2876,6 +3053,22 @@
         assert isinstance(reg, RegLoc)
         self.mc.MOV_rr(reg.value, ebp.value)
 
+    def genop_discard_stm_set_revision_gc(self, op, arglocs):
+        base_loc, ofs_loc, size_loc = arglocs
+        assert isinstance(size_loc, ImmedLoc)
+        mc = self.mc
+
+        if IS_X86_32:
+            todo()
+            
+        rn = self._get_stm_private_rev_num_addr()
+        self._tl_segment_if_stm(mc)
+        mc.MOV_rj(X86_64_SCRATCH_REG.value, rn)
+
+        dest_addr = AddressLoc(base_loc, ofs_loc)
+        mc.MOV(dest_addr, X86_64_SCRATCH_REG)
+
+        
     def genop_stm_transaction_break(self, op, arglocs, result_loc):
         assert self.cpu.gc_ll_descr.stm
         if not we_are_translated():
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -849,8 +849,6 @@
 
     def consider_call_malloc_nursery(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        assert gc_ll_descr.get_malloc_slowpath_addr() is not None
-        # ^^^ if this returns None, don't translate the rest of this function
         #
         size_box = op.getarg(0)
         assert isinstance(size_box, ConstInt)
@@ -865,15 +863,16 @@
         gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before*
         self.rm.possibly_free_var(tmp_box)
         #
-        self.assembler.malloc_cond(
-            gc_ll_descr.get_nursery_free_addr(),
-            gc_ll_descr.get_nursery_top_addr(),
-            size, gcmap)
+        if gc_ll_descr.stm:
+            self.assembler.malloc_cond_stm(size, gcmap)
+        else:
+            self.assembler.malloc_cond(
+                gc_ll_descr.get_nursery_free_addr(),
+                gc_ll_descr.get_nursery_top_addr(),
+                size, gcmap)
 
     def consider_call_malloc_nursery_varsize_frame(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        assert gc_ll_descr.get_malloc_slowpath_addr() is not None
-        # ^^^ if this returns None, don't translate the rest of this function
         #
         size_box = op.getarg(0)
         assert isinstance(size_box, BoxInt) # we cannot have a const here!
@@ -889,11 +888,13 @@
         gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before*
         self.rm.possibly_free_var(tmp_box)
         #
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        self.assembler.malloc_cond_varsize_frame(
-            gc_ll_descr.get_nursery_free_addr(),
-            gc_ll_descr.get_nursery_top_addr(),
-            sizeloc, gcmap)
+        if gc_ll_descr.stm:
+            self.assembler.malloc_cond_varsize_frame_stm(sizeloc, gcmap)
+        else:
+            self.assembler.malloc_cond_varsize_frame(
+                gc_ll_descr.get_nursery_free_addr(),
+                gc_ll_descr.get_nursery_top_addr(),
+                sizeloc, gcmap)
 
     def consider_call_malloc_nursery_varsize(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
@@ -919,11 +920,16 @@
         #
         itemsize = op.getarg(1).getint()
         maxlength = (gc_ll_descr.max_size_of_young_obj - WORD * 2) / itemsize
-        self.assembler.malloc_cond_varsize(
-            op.getarg(0).getint(),
-            gc_ll_descr.get_nursery_free_addr(),
-            gc_ll_descr.get_nursery_top_addr(),
-            lengthloc, itemsize, maxlength, gcmap, arraydescr)
+        if gc_ll_descr.stm:
+            self.assembler.malloc_cond_varsize_stm(
+                op.getarg(0).getint(), 
+                lengthloc, itemsize, maxlength, gcmap, arraydescr)
+        else:
+            self.assembler.malloc_cond_varsize(
+                op.getarg(0).getint(),
+                gc_ll_descr.get_nursery_free_addr(),
+                gc_ll_descr.get_nursery_top_addr(),
+                lengthloc, itemsize, maxlength, gcmap, arraydescr)
 
     def get_gcmap(self, forbidden_regs=[], noregs=False):
         frame_depth = self.fm.get_frame_depth()
@@ -1267,6 +1273,16 @@
                 if isinstance(loc, FrameLoc):
                     self.fm.hint_frame_locations[box] = loc
 
+    
+    def consider_stm_set_revision_gc(self, op):
+        ofs, size, _ = unpack_fielddescr(op.getdescr())
+        ofs_loc = imm(ofs)
+        size_loc = imm(size)
+        assert isinstance(size_loc, ImmedLoc)
+        args = op.getarglist()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        self.perform_discard(op, [base_loc, ofs_loc, size_loc])
+        
     def consider_stm_transaction_break(self, op):
         # XXX use the extra 3 words in the stm resume buffer to save
         # up to 3 registers, too.  For now we just flush them all.
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -348,6 +348,7 @@
                          rop.CALL_MALLOC_NURSERY_VARSIZE_FRAME,
                          rop.LABEL,
                          rop.STM_TRANSACTION_BREAK,
+                         rop.STM_SET_REVISION_GC,
                          ):      # list of opcodes never executed by pyjitpl
                 continue
             raise AssertionError("missing %r" % (key,))
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -511,6 +511,7 @@
     'RECORD_KNOWN_CLASS/2',   # [objptr, clsptr]
     'KEEPALIVE/1',
     'STM_TRANSACTION_BREAK/0',
+    'STM_SET_REVISION_GC/1d', # not really GC, writes raw to the header
 
     '_CANRAISE_FIRST', # ----- start of can_raise operations -----
     '_CALL_FIRST',
diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -9,6 +9,7 @@
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.debug import ll_assert
 from rpython.rlib.rarithmetic import LONG_BIT, r_uint
+from rpython.rtyper.extregistry import ExtRegistryEntry
 
 WORD = LONG_BIT // 8
 NULL = llmemory.NULL
@@ -36,8 +37,11 @@
     malloc_zero_filled = True
     #gcflag_extra = GCFLAG_EXTRA
 
-
-    GCHDR = lltype.GcStruct(
+    # SYNC with et.h
+    TD_NURSERY_CURRENT = 80
+    TD_NURSERY_NEXTLIMIT = 88
+    
+    GCHDR = lltype.Struct(
         'GCPTR',
         ('h_tid', lltype.Unsigned),
         ('h_revision', lltype.Signed),
@@ -79,6 +83,13 @@
     
     FX_MASK = 65535
 
+    # keep in sync with nursery.h:
+    
+    # maximum size of object in nursery (is actually dependent on
+    # nursery size, but this should work)
+    GC_NURSERY_SECTION = 135168
+    
+
     def get_type_id(self, obj):
         return llop.stm_get_tid(llgroup.HALFWORD, obj)
 
@@ -151,8 +162,7 @@
 
     @classmethod
     def JIT_max_size_of_young_obj(cls):
-        # XXX there is actually a maximum, check
-        return None
+        return cls.GC_NURSERY_SECTION
 
     @classmethod
     def JIT_minimal_size_in_nursery(cls):
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -5,6 +5,11 @@
 from rpython.rlib.jit import dont_look_inside
 
 @dont_look_inside
+def get_thread_descriptor_adr():
+    addr = llop.stm_get_adr_of_thread_descriptor(llmemory.Address)
+    return rffi.cast(lltype.Signed, addr)
+
+ at dont_look_inside
 def get_adr_of_private_rev_num():
     addr = llop.stm_get_adr_of_private_rev_num(llmemory.Address)
     return rffi.cast(lltype.Signed, addr)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -951,6 +951,7 @@
     op_stm_barrier = _stm_not_implemented
     op_stm_push_root = _stm_not_implemented
     op_stm_pop_root_into = _stm_not_implemented
+    op_stm_get_adr_of_thread_descriptor = _stm_not_implemented
     op_stm_get_adr_of_read_barrier_cache = _stm_not_implemented
     op_stm_get_adr_of_private_rev_num = _stm_not_implemented
     op_stm_enter_callback_call = _stm_not_implemented
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -448,6 +448,7 @@
 
     'stm_get_adr_of_private_rev_num':LLOp(),
     'stm_get_adr_of_read_barrier_cache':LLOp(),
+    'stm_get_adr_of_thread_descriptor': LLOp(),
 
     'stm_ignored_start':      LLOp(canrun=True),
     'stm_ignored_stop':       LLOp(canrun=True),
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -590,6 +590,7 @@
     OP_STM_PTR_EQ                       = _OP_STM
     OP_STM_PUSH_ROOT                    = _OP_STM
     OP_STM_POP_ROOT_INTO                = _OP_STM
+    OP_STM_GET_ADR_OF_THREAD_DESCRIPTOR = _OP_STM
     OP_STM_GET_ROOT_STACK_TOP           = _OP_STM
     OP_STM_GET_ADR_OF_PRIVATE_REV_NUM   = _OP_STM
     OP_STM_GET_ADR_OF_READ_BARRIER_CACHE= _OP_STM
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -114,6 +114,11 @@
     return '%s = (%s)stm_pop_root();' % (
         arg0, cdecl(funcgen.lltypename(op.args[0]), ''))
 
+def stm_get_adr_of_thread_descriptor(funcgen, op):
+    result = funcgen.expr(op.result)
+    return '%s = (%s)&thread_descriptor;' % (
+        result, cdecl(funcgen.lltypename(op.result), ''))
+    
 def stm_get_root_stack_top(funcgen, op):
     result = funcgen.expr(op.result)
     return '%s = (%s)&stm_shadowstack;' % (

From noreply at buildbot.pypy.org  Wed Oct 16 18:38:13 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 18:38:13 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: fix this test
Message-ID: <20131016163813.CE0801D22DC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67431:06e3dae26511
Date: 2013-10-16 17:53 +0200
http://bitbucket.org/pypy/pypy/changeset/06e3dae26511/

Log:	fix this test

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -68,7 +68,6 @@
             locs = None
         else:
             self.guard_failed = True
-            xxxx
         if result_type == 'int':
             return BoxInt(self.cpu.get_int_value(deadframe, locs, 0))
         elif result_type == 'ref':
@@ -100,7 +99,6 @@
                       ResOperation(rop.FINISH, results, None,
                                    descr=BasicFinalDescr(0))]
         if operations[0].is_guard():
-            operations[0].setfailargs([])
             if not descr:
                 descr = BasicFailDescr(1)
         if descr is not None:
@@ -1187,9 +1185,12 @@
             #
             ks = range(nb_args)
             random.shuffle(ks)
+            intboxes = 0
+            floatboxes = 0
             for k in ks:
                 if isinstance(inputargs[k], BoxInt):
                     newbox = BoxInt()
+                    intboxes += 1
                     x = r.randrange(-100000, 100000)
                     operations.append(
                         ResOperation(rop.INT_ADD, [inputargs[k],
@@ -1198,6 +1199,7 @@
                     y = values[k] + x
                 else:
                     newbox = BoxFloat()
+                    floatboxes += 1
                     x = r.random()
                     operations.append(
                         ResOperation(rop.FLOAT_ADD, [inputargs[k],
@@ -1210,12 +1212,32 @@
                 retvalues.insert(kk, y)
             #
             zero = BoxInt()
+            jitcode = JitCode("name")
+            jitcode.setup(num_regs_i=intboxes, num_regs_r=0,
+                          num_regs_f=floatboxes)
             operations.extend([
+                ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None,
+                             descr=jitcode),
                 ResOperation(rop.SAME_AS, [ConstInt(0)], zero),
+            ])
+            i_int = 0
+            i_float = intboxes
+            expvalues = [None] * len(retboxes)
+            for i, box in enumerate(retboxes):
+                if isinstance(box, BoxInt):
+                    pos = i_int
+                    i_int += 1
+                else:
+                    pos = i_float
+                    i_float += 1
+                operations.append(ResOperation(rop.RESUME_PUT, [box, ConstInt(0), ConstInt(pos)], None))
+                expvalues[pos] = retvalues[i]
+            operations.extend([
                 ResOperation(rop.GUARD_TRUE, [zero], None, descr=guarddescr),
+                ResOperation(rop.LEAVE_FRAME, [], None),
                 ResOperation(rop.FINISH, [], None, descr=faildescr)
                 ])
-            operations[-2].setfailargs(retboxes)
+            #operations[-2].setfailargs(retboxes)
             print inputargs
             for op in operations:
                 print op
@@ -1223,14 +1245,15 @@
             #
             deadframe = self.cpu.execute_token(looptoken, *values)
             fail = self.cpu.get_latest_descr(deadframe)
+            locs = rebuild_locs_from_resumedata(fail)
             assert fail.identifier == 42
             #
-            for k in range(len(retvalues)):
-                if isinstance(retboxes[k], BoxInt):
-                    got = self.cpu.get_int_value(deadframe, k)
-                else:
-                    got = self.cpu.get_float_value(deadframe, k)
-                assert got == retvalues[k]
+            for k in range(intboxes):
+                got = self.cpu.get_int_value(deadframe, locs, k)
+                assert got == expvalues[k]
+            for k in range(floatboxes):
+                got = self.cpu.get_float_value(deadframe, locs, k + intboxes)
+                assert got == expvalues[k + intboxes]
 
     def test_jump(self):
         # this test generates small loops where the JUMP passes many
diff --git a/rpython/jit/metainterp/resume2.py b/rpython/jit/metainterp/resume2.py
--- a/rpython/jit/metainterp/resume2.py
+++ b/rpython/jit/metainterp/resume2.py
@@ -59,7 +59,8 @@
             xxx
         else:
             assert frontend_position < jitcode.num_regs()
-            xxx
+            self.put_box_float(frame, frontend_position - jitcode.num_regs_r()
+                               - jitcode.num_regs_i(), box)
 
 class DirectResumeReader(AbstractResumeReader):
     pass
@@ -84,6 +85,9 @@
     def put_box_int(self, frame, position, box):
         frame.registers_i[position] = box
 
+    def put_box_float(self, frame, position, box):
+        frame.registers_f[position] = box
+
     def finish(self):
         cpu = self.metainterp.cpu
         for box, position in self.backend_values.iteritems():

From noreply at buildbot.pypy.org  Wed Oct 16 18:38:15 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 18:38:15 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: (fijal,
	arigo) refactor test_jump
Message-ID: <20131016163815.0E42C1D22E7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67432:17dfffc116ff
Date: 2013-10-16 18:25 +0200
http://bitbucket.org/pypy/pypy/changeset/17dfffc116ff/

Log:	(fijal, arigo) refactor test_jump

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1270,14 +1270,21 @@
             print 'Passing %d arguments around...' % nb_args
             #
             inputargs = []
+            floatargs = []
+            intargs = []
+            refargs = []
             for k in range(nb_args):
                 kind = r.randrange(0, numkinds)
                 if kind == 0:
-                    inputargs.append(BoxInt())
+                    box = BoxInt()
+                    intargs.append(box)
                 elif kind == 1:
-                    inputargs.append(BoxPtr())
+                    box = BoxPtr()
+                    refargs.append(box)
                 else:
-                    inputargs.append(BoxFloat())
+                    box = BoxFloat()
+                    floatargs.append(box)
+                inputargs.append(box)
             jumpargs = []
             remixing = []
             for srcbox in inputargs:
@@ -1289,25 +1296,46 @@
                     otherbox = srcbox
                 jumpargs.append(otherbox)
             #
-            index_counter = r.randrange(0, len(inputargs)+1)
+            index_counter = 0 #r.randrange(0, len(inputargs)+1)
             i0 = BoxInt()
             i1 = BoxInt()
             i2 = BoxInt()
             inputargs.insert(index_counter, i0)
+            intargs.insert(index_counter, i0)
             jumpargs.insert(index_counter, i1)
             #
             looptoken = JitCellToken()
             targettoken = TargetToken()
             faildescr = BasicFailDescr(15)
+            jitcode = JitCode("jitcode")
+            jitcode.setup(num_regs_i=len(intargs), num_regs_r=len(refargs),
+                          num_regs_f=len(floatargs))
             operations = [
+                ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None,
+                             descr=jitcode),
                 ResOperation(rop.LABEL, inputargs, None, descr=targettoken),
                 ResOperation(rop.INT_SUB, [i0, ConstInt(1)], i1),
                 ResOperation(rop.INT_GE, [i1, ConstInt(0)], i2),
+            ]
+            for i in range(len(intargs)):
+                operations.append(ResOperation(rop.RESUME_PUT,
+                                               [intargs[i], ConstInt(0),
+                                                ConstInt(i)], None))
+            for i in range(len(refargs)):
+                pos = i + len(intargs)
+                operations.append(ResOperation(rop.RESUME_PUT,
+                                               [refargs[i], ConstInt(0),
+                                                ConstInt(pos)], None))
+            for i in range(len(floatargs)):
+                pos = i + len(intargs) + len(refargs)
+                operations.append(ResOperation(rop.RESUME_PUT,
+                                               [floatargs[i], ConstInt(0),
+                                                ConstInt(pos)], None))
+            operations.extend([
                 ResOperation(rop.GUARD_TRUE, [i2], None),
                 ResOperation(rop.JUMP, jumpargs, None, descr=targettoken),
-                ]
-            operations[3].setfailargs(inputargs[:])
-            operations[3].setdescr(faildescr)
+                ])
+            operations[-2].setdescr(faildescr)
             #
             self.cpu.compile_loop(None, inputargs, operations, looptoken)
             #
@@ -1315,14 +1343,15 @@
             S = lltype.GcStruct('S')
             for box in inputargs:
                 if isinstance(box, BoxInt):
-                    values.append(r.randrange(-10000, 10000))
+                    v = r.randrange(-10000, 10000)
                 elif isinstance(box, BoxPtr):
                     p = lltype.malloc(S)
-                    values.append(lltype.cast_opaque_ptr(llmemory.GCREF, p))
+                    v = lltype.cast_opaque_ptr(llmemory.GCREF, p)
                 elif isinstance(box, BoxFloat):
-                    values.append(longlong.getfloatstorage(r.random()))
+                    v = longlong.getfloatstorage(r.random())
                 else:
                     assert 0
+                values.append(v)
             values[index_counter] = 11
             #
             deadframe = self.cpu.execute_token(looptoken, *values)
@@ -1339,16 +1368,26 @@
             #
             assert dstvalues[index_counter] == 11
             dstvalues[index_counter] = 0
-            for i, (box, val) in enumerate(zip(inputargs, dstvalues)):
-                if isinstance(box, BoxInt):
-                    got = self.cpu.get_int_value(deadframe, i)
-                elif isinstance(box, BoxPtr):
-                    got = self.cpu.get_ref_value(deadframe, i)
-                elif isinstance(box, BoxFloat):
-                    got = self.cpu.get_float_value(deadframe, i)
+            locs = rebuild_locs_from_resumedata(fail)
+            intvals = []
+            refvals = []
+            floatvals = []
+            for val in dstvalues:
+                if isinstance(val, int):
+                    intvals.append(val)
+                elif isinstance(val, float):
+                    floatvals.append(val)
                 else:
-                    assert 0
-                assert type(got) == type(val)
+                    refvals.append(val)
+            for i, val in enumerate(intvals):
+                got = self.cpu.get_int_value(deadframe, locs, i)
+                assert got == val
+            for i, val in enumerate(refvals):
+                got = self.cpu.get_ref_value(deadframe, locs, i + len(intvals))
+                assert got == val
+            for i, val in enumerate(floatvals):
+                got = self.cpu.get_float_value(deadframe, locs,
+                                               i + len(intvals) + len(refvals))
                 assert got == val
 
     def test_compile_bridge_float(self):
diff --git a/rpython/jit/metainterp/resume2.py b/rpython/jit/metainterp/resume2.py
--- a/rpython/jit/metainterp/resume2.py
+++ b/rpython/jit/metainterp/resume2.py
@@ -56,7 +56,8 @@
         if frontend_position < jitcode.num_regs_i():
             self.put_box_int(frame, frontend_position, box)
         elif frontend_position < (jitcode.num_regs_r() + jitcode.num_regs_i()):
-            xxx
+            self.put_box_ref(frame, frontend_position - jitcode.num_regs_i(),
+                             box)
         else:
             assert frontend_position < jitcode.num_regs()
             self.put_box_float(frame, frontend_position - jitcode.num_regs_r()
@@ -85,6 +86,9 @@
     def put_box_int(self, frame, position, box):
         frame.registers_i[position] = box
 
+    def put_box_ref(self, frame, position, box):
+        frame.registers_r[position] = box
+
     def put_box_float(self, frame, position, box):
         frame.registers_f[position] = box
 

From noreply at buildbot.pypy.org  Wed Oct 16 18:43:57 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 18:43:57 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: Fix on 32-bit
Message-ID: <20131016164357.AFBC81D22DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: resume-refactor
Changeset: r67433:707143609165
Date: 2013-10-16 18:43 +0200
http://bitbucket.org/pypy/pypy/changeset/707143609165/

Log:	Fix on 32-bit

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1375,7 +1375,7 @@
             for val in dstvalues:
                 if isinstance(val, int):
                     intvals.append(val)
-                elif isinstance(val, float):
+                elif isinstance(val, longlong.r_float_storage):
                     floatvals.append(val)
                 else:
                     refvals.append(val)

From noreply at buildbot.pypy.org  Wed Oct 16 18:56:36 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 18:56:36 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: Next test
Message-ID: <20131016165636.BFD191D22DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: resume-refactor
Changeset: r67434:80e2ea27c994
Date: 2013-10-16 18:49 +0200
http://bitbucket.org/pypy/pypy/changeset/80e2ea27c994/

Log:	Next test

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1399,15 +1399,23 @@
         faildescr1 = BasicFailDescr(1)
         faildescr2 = BasicFailDescr(2)
         faildescr3 = BasicFinalDescr(3)
+        jitcode = JitCode("jitcode")
+        jitcode.setup(num_regs_i=0, num_regs_r=0, num_regs_f=12)
         operations = [
+            ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None, descr=jitcode),
             ResOperation(rop.LABEL, fboxes, None, descr=targettoken),
             ResOperation(rop.FLOAT_LE, [fboxes[0], constfloat(9.2)], i2),
+            ]
+        for i in range(12):
+            operations.append(
+                ResOperation(rop.RESUME_PUT, [fboxes[i], ConstInt(0),
+                                              ConstInt(i)], None))
+        operations.extend([
             ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1),
             ResOperation(rop.GUARD_FALSE, [i2], None, descr=faildescr2),
+            ResOperation(rop.LEAVE_FRAME, [], None),
             ResOperation(rop.FINISH, [], None, descr=faildescr3),
-            ]
-        operations[-3].setfailargs(fboxes)
-        operations[-2].setfailargs(fboxes)
+            ])
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, fboxes, operations, looptoken)
 
@@ -1418,7 +1426,9 @@
             ResOperation(rop.JUMP, [f3]+fboxes2[1:], None, descr=targettoken),
         ]
 
-        self.cpu.compile_bridge(None, faildescr1, fboxes2, bridge, looptoken)
+        self.cpu.compile_bridge(None, faildescr1, fboxes2,
+                                rebuild_locs_from_resumedata(faildescr1),
+                                bridge, looptoken)
 
         args = []
         for i in range(len(fboxes)):
@@ -1427,11 +1437,12 @@
         deadframe = self.cpu.execute_token(looptoken, *args)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 2
-        res = self.cpu.get_float_value(deadframe, 0)
+        locs = rebuild_locs_from_resumedata(fail)
+        res = self.cpu.get_float_value(deadframe, locs, 0)
         assert longlong.getrealfloat(res) == 8.5
         for i in range(1, len(fboxes)):
             got = longlong.getrealfloat(self.cpu.get_float_value(
-                deadframe, i))
+                deadframe, locs, i))
             assert got == 13.5 + 6.73 * i
 
     def test_compile_bridge_spilled_float(self):

From noreply at buildbot.pypy.org  Wed Oct 16 18:56:37 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 18:56:37 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: Next test
Message-ID: <20131016165637.EC10B1D22DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: resume-refactor
Changeset: r67435:c0ff29e897ae
Date: 2013-10-16 18:55 +0200
http://bitbucket.org/pypy/pypy/changeset/c0ff29e897ae/

Log:	Next test

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1450,16 +1450,22 @@
             py.test.skip("requires floats")
         fboxes = [BoxFloat() for i in range(3)]
         faildescr1 = BasicFailDescr(100)
-        faildescr2 = BasicFinalDescr(102)
+        jitcode = JitCode("jitcode")
+        jitcode.setup(num_regs_i=0, num_regs_r=0, num_regs_f=3)
         loopops = """
-        [i0,f1, f2]
+        [i0, f1, f2]
+        enter_frame(-1, descr=jitcode)
         f3 = float_add(f1, f2)
         force_spill(f3)
         force_spill(f1)
         force_spill(f2)
-        guard_false(i0) [f1, f2, f3]
+        resume_put(f1, 0, 0)
+        resume_put(f2, 0, 1)
+        resume_put(f3, 0, 2)
+        guard_false(i0, descr=faildescr1)
+        leave_frame()
         finish()"""
-        loop = parse(loopops)
+        loop = parse(loopops, namespace=locals())
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken)
         args = [1]
@@ -1467,32 +1473,36 @@
         args.append(longlong.getfloatstorage(0.75))
         deadframe = self.cpu.execute_token(looptoken, *args)  #xxx check
         fail = self.cpu.get_latest_descr(deadframe)
-        assert loop.operations[-2].getdescr() == fail
-        f1 = self.cpu.get_float_value(deadframe, 0)
-        f2 = self.cpu.get_float_value(deadframe, 1)
-        f3 = self.cpu.get_float_value(deadframe, 2)
+        assert loop.operations[-3].getdescr() is fail is faildescr1
+        locs = rebuild_locs_from_resumedata(fail)
+        f1 = self.cpu.get_float_value(deadframe, locs, 0)
+        f2 = self.cpu.get_float_value(deadframe, locs, 1)
+        f3 = self.cpu.get_float_value(deadframe, locs, 2)
         assert longlong.getrealfloat(f1) == 132.25
         assert longlong.getrealfloat(f2) == 0.75
         assert longlong.getrealfloat(f3) == 133.0
 
+        faildescr2 = BasicFinalDescr(102)
+        faildescr3 = BasicFailDescr(103)
         zero = BoxInt()
         bridgeops = [
             ResOperation(rop.SAME_AS, [ConstInt(0)], zero),
-            ResOperation(rop.GUARD_TRUE, [zero], None, descr=faildescr1),
+            ResOperation(rop.GUARD_TRUE, [zero], None, descr=faildescr3),
+            ResOperation(rop.LEAVE_FRAME, [], None),
             ResOperation(rop.FINISH, [], None, descr=faildescr2),
             ]
-        bridgeops[-2].setfailargs(fboxes[:])
-        self.cpu.compile_bridge(None, loop.operations[-2].getdescr(), fboxes,
-                                                        bridgeops, looptoken)
+        self.cpu.compile_bridge(None, faildescr1, fboxes,
+                                locs, bridgeops, looptoken)
         args = [1,
                 longlong.getfloatstorage(132.25),
                 longlong.getfloatstorage(0.75)]
         deadframe = self.cpu.execute_token(looptoken, *args)
         fail = self.cpu.get_latest_descr(deadframe)
-        assert fail.identifier == 100
-        f1 = self.cpu.get_float_value(deadframe, 0)
-        f2 = self.cpu.get_float_value(deadframe, 1)
-        f3 = self.cpu.get_float_value(deadframe, 2)
+        assert fail.identifier == 103
+        locs = rebuild_locs_from_resumedata(fail)
+        f1 = self.cpu.get_float_value(deadframe, locs, 0)
+        f2 = self.cpu.get_float_value(deadframe, locs, 1)
+        f3 = self.cpu.get_float_value(deadframe, locs, 2)
         assert longlong.getrealfloat(f1) == 132.25
         assert longlong.getrealfloat(f2) == 0.75
         assert longlong.getrealfloat(f3) == 133.0

From noreply at buildbot.pypy.org  Wed Oct 16 18:58:33 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 16 Oct 2013 18:58:33 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: More tests
Message-ID: <20131016165833.D4E981D22DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: resume-refactor
Changeset: r67436:1a003348040a
Date: 2013-10-16 18:57 +0200
http://bitbucket.org/pypy/pypy/changeset/1a003348040a/

Log:	More tests

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1525,7 +1525,6 @@
                     ResOperation(opguard, [res], None, descr=faildescr1),
                     ResOperation(rop.FINISH, [], None, descr=faildescr2),
                     ]
-                operations[1].setfailargs([])
                 looptoken = JitCellToken()
                 self.cpu.compile_loop(None, inputargs, operations, looptoken)
                 #
@@ -1570,7 +1569,6 @@
                         ResOperation(opguard, [b1], None, descr=faildescr1),
                         ResOperation(rop.FINISH, [], None, descr=faildescr2),
                         ]
-                    operations[-2].setfailargs([])
                     looptoken = JitCellToken()
                     self.cpu.compile_loop(None, inputargs, operations, looptoken)
                     #
@@ -1622,7 +1620,6 @@
                         ResOperation(opguard, [b1], None, descr=faildescr1),
                         ResOperation(rop.FINISH, [], None, descr=faildescr2),
                         ]
-                    operations[-2].setfailargs([])
                     looptoken = JitCellToken()
                     self.cpu.compile_loop(None, inputargs, operations, looptoken)
                     #
@@ -1678,7 +1675,6 @@
                         ResOperation(opguard, [b1], None, descr=faildescr1),
                         ResOperation(rop.FINISH, [], None, descr=faildescr2),
                         ]
-                    operations[-2].setfailargs([])
                     looptoken = JitCellToken()
                     self.cpu.compile_loop(None, inputargs, operations, looptoken)
                     #
@@ -1808,7 +1804,6 @@
                                          descr=BasicFailDescr(4)),
                             ResOperation(rop.FINISH, [], None,
                                          descr=BasicFinalDescr(5))]
-                        operations[1].setfailargs([])
                         looptoken = JitCellToken()
                         # Use "set" to unique-ify inputargs
                         unique_testcase_list = list(set(testcase))

From noreply at buildbot.pypy.org  Wed Oct 16 22:29:36 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 22:29:36 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: fix more tests
Message-ID: <20131016202936.D63CF1D22EB@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67437:59ad21475a57
Date: 2013-10-16 22:21 +0200
http://bitbucket.org/pypy/pypy/changeset/59ad21475a57/

Log:	fix more tests

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -10,6 +10,8 @@
     OrderedDict = dict # too bad
 
 class TempBox(Box):
+    type = 't' # none of the types
+    
     def __init__(self):
         pass
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2096,15 +2096,21 @@
     def test_exceptions(self):
         exc_tp = None
         exc_ptr = None
+        faildescr = BasicFailDescr(1)
         def func(i):
             if i:
                 raise LLException(exc_tp, exc_ptr)
 
+        jitcode = JitCode("name")
+        jitcode.setup(num_regs_i=1, num_regs_r=0, num_regs_f=0)
         ops = '''
         [i0]
+        enter_frame(-1, descr=jitcode)
         i1 = same_as(1)
         call(ConstClass(fptr), i0, descr=calldescr)
-        p0 = guard_exception(ConstClass(xtp)) [i1]
+        resume_put(i1, 0, 0)
+        p0 = guard_exception(ConstClass(xtp), descr=faildescr)
+        leave_frame()
         finish(p0)
         '''
         FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void))
@@ -2126,11 +2132,12 @@
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 1)
-        assert self.cpu.get_ref_value(deadframe, 0) == xptr
+        assert self.cpu.get_ref_value(deadframe, None, 0) == xptr
         excvalue = self.cpu.grab_exc_value(deadframe)
         assert not excvalue
         deadframe = self.cpu.execute_token(looptoken, 0)
-        assert self.cpu.get_int_value(deadframe, 0) == 1
+        locs = rebuild_locs_from_resumedata(faildescr)
+        assert self.cpu.get_int_value(deadframe, locs, 0) == 1
         excvalue = self.cpu.grab_exc_value(deadframe)
         assert not excvalue
 
@@ -2149,7 +2156,7 @@
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 1)
-        assert self.cpu.get_int_value(deadframe, 0) == 1
+        assert self.cpu.get_int_value(deadframe, locs, 0) == 1
         excvalue = self.cpu.grab_exc_value(deadframe)
         assert excvalue == yptr
 
@@ -2157,20 +2164,24 @@
         exc_ptr = xptr
         ops = '''
         [i0]
+        enter_frame(-1, descr=jitcode)
         i1 = same_as(1)
         call(ConstClass(fptr), i0, descr=calldescr)
-        guard_no_exception() [i1]
+        resume_put(i0, 0, 0)
+        guard_no_exception(descr=faildescr)
+        leave_frame()
         finish(0)
         '''
         loop = parse(ops, self.cpu, namespace=locals())
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 1)
-        assert self.cpu.get_int_value(deadframe, 0) == 1
+        locs = rebuild_locs_from_resumedata(faildescr)
+        assert self.cpu.get_int_value(deadframe, locs, 0) == 1
         excvalue = self.cpu.grab_exc_value(deadframe)
         assert excvalue == xptr
         deadframe = self.cpu.execute_token(looptoken, 0)
-        assert self.cpu.get_int_value(deadframe, 0) == 0
+        assert self.cpu.get_int_value(deadframe, locs, 0) == 0
         excvalue = self.cpu.grab_exc_value(deadframe)
         assert not excvalue
 

From noreply at buildbot.pypy.org  Wed Oct 16 22:29:38 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 22:29:38 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: port another test
Message-ID: <20131016202938.31C841D22EC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67438:9ab347f35853
Date: 2013-10-16 22:28 +0200
http://bitbucket.org/pypy/pypy/changeset/9ab347f35853/

Log:	port another test

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2345,30 +2345,48 @@
             calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
                                              EffectInfo.MOST_GENERAL)
 
+            # [i1, i2, i3, i4, i5, i6, f0, f1]
+            jitcode = JitCode('jitcode')
+            jitcode.setup(num_regs_i=6, num_regs_r=0, num_regs_f=2)
+            faildescr = BasicFailDescr()
             ops = '''
             [i0, i1, i2, i3, i4, i5, i6, f0, f1]
+            enter_frame(-1, descr=jitcode)
+            resume_put(i1, 0, 0)
+            resume_put(i2, 0, 1)
+            resume_put(i3, 0, 2)
+            resume_put(i4, 0, 3)
+            resume_put(i5, 0, 4)
+            resume_put(i6, 0, 5)
+            resume_put(f0, 0, 6)
+            resume_put(f1, 0, 7)
             cond_call(i1, ConstClass(func_ptr), %s)
-            guard_false(i0, descr=faildescr) [i1, i2, i3, i4, i5, i6, f0, f1]
-            ''' % ', '.join(['i%d' % (j + 2) for j in range(i)] + ["descr=calldescr"])
-            loop = parse(ops, namespace={'faildescr': BasicFailDescr(),
+            guard_false(i0, descr=faildescr)
+            leave_frame()
+            ''' % (
+                ', '.join(['i%d' % (j + 2) for j in range(i)] +
+                          ["descr=calldescr"]))
+            loop = parse(ops, namespace={'faildescr': faildescr,
                                          'func_ptr': func_ptr,
-                                         'calldescr': calldescr})
+                                         'calldescr': calldescr,
+                                         'jitcode': jitcode})
             looptoken = JitCellToken()
             self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken)
             f1 = longlong.getfloatstorage(1.2)
             f2 = longlong.getfloatstorage(3.4)
             frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5, f1, f2)
             assert not called
+            locs = rebuild_locs_from_resumedata(faildescr)
             for j in range(5):
-                assert self.cpu.get_int_value(frame, j) == j
-            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 6)) == 1.2
-            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 3.4
+                assert self.cpu.get_int_value(frame, locs, j) == j
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, locs, 6)) == 1.2
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, locs, 7)) == 3.4
             frame = self.cpu.execute_token(looptoken, 1, 1, 1, 2, 3, 4, 5, f1, f2)
             assert called == [tuple(range(1, i + 1))]
             for j in range(4):
-                assert self.cpu.get_int_value(frame, j + 1) == j + 1
-            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 6)) == 1.2
-            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 3.4
+                assert self.cpu.get_int_value(frame, locs, j + 1) == j + 1
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, locs, 6)) == 1.2
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, locs, 7)) == 3.4
 
     def test_force_operations_returning_void(self):
         values = []

From noreply at buildbot.pypy.org  Wed Oct 16 22:34:05 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 22:34:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix ndarray.item() exception in one case
Message-ID: <20131016203405.A06001D22EB@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67439:bc9dfce89505
Date: 2013-10-16 16:32 -0400
http://bitbucket.org/pypy/pypy/changeset/bc9dfce89505/

Log:	fix ndarray.item() exception in one case

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -421,8 +421,8 @@
                                      [0] * len(self.get_shape()))
                 assert isinstance(w_obj, interp_boxes.W_GenericBox)
                 return w_obj.item(space)
-            raise OperationError(space.w_IndexError,
-                                 space.wrap("index out of bounds"))
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("can only convert an array of size 1 to a Python scalar"))
         if space.isinstance_w(w_arg, space.w_int):
             if self.is_scalar():
                 raise OperationError(space.w_IndexError,
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2486,7 +2486,7 @@
         assert type(array(True).item()) is bool
         assert type(array(3.5).item()) is float
         raises(IndexError, "array(3).item(15)")
-        raises(IndexError, "array([1, 2, 3]).item()")
+        raises(ValueError, "array([1, 2, 3]).item()")
         assert array([3]).item(0) == 3
         assert type(array([3]).item(0)) is int
         assert array([1, 2, 3]).item(-1) == 3

From noreply at buildbot.pypy.org  Wed Oct 16 23:12:19 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 23:12:19 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: another test
Message-ID: <20131016211219.5B04C1D22DC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67440:c5180024bb6f
Date: 2013-10-16 22:48 +0200
http://bitbucket.org/pypy/pypy/changeset/c5180024bb6f/

Log:	another test

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2393,9 +2393,11 @@
         def maybe_force(token, flag):
             if flag:
                 deadframe = self.cpu.force(token)
-                values.append(self.cpu.get_latest_descr(deadframe))
-                values.append(self.cpu.get_int_value(deadframe, 0))
-                values.append(self.cpu.get_int_value(deadframe, 1))
+                fail = self.cpu.get_latest_descr(deadframe)
+                locs = rebuild_locs_from_resumedata(fail)
+                values.append(fail)
+                values.append(self.cpu.get_int_value(deadframe, locs, 0))
+                values.append(self.cpu.get_int_value(deadframe, locs, 1))
                 self.cpu.set_savedata_ref(deadframe, random_gcref)
 
         FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Void)
@@ -2408,27 +2410,32 @@
         i1 = BoxInt()
         tok = BoxPtr()
         faildescr = BasicFailDescr(1)
+        jitcode = JitCode('name')
+        jitcode.setup(num_regs_i=2, num_regs_r=0, num_regs_f=0)
         ops = [
-        ResOperation(rop.FORCE_TOKEN, [], tok),
-        ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], None,
-                     descr=calldescr),
-        ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
-        ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(0))
+            ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None, descr=jitcode),
+            ResOperation(rop.FORCE_TOKEN, [], tok),
+            ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], None,
+                         descr=calldescr),
+            ResOperation(rop.RESUME_PUT, [i0, ConstInt(0), ConstInt(1)], None),
+            ResOperation(rop.RESUME_PUT, [i1, ConstInt(0), ConstInt(0)], None),
+            ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
+            ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(0))
         ]
-        ops[2].setfailargs([i1, i0])
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, [i0, i1], ops, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 20, 0)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 0
-        assert self.cpu.get_int_value(deadframe, 0) == 20
+        assert self.cpu.get_int_value(deadframe, None, 0) == 20
         assert values == []
 
         deadframe = self.cpu.execute_token(looptoken, 10, 1)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 1
-        assert self.cpu.get_int_value(deadframe, 0) == 1
-        assert self.cpu.get_int_value(deadframe, 1) == 10
+        locs = rebuild_locs_from_resumedata(fail)
+        assert self.cpu.get_int_value(deadframe, locs, 0) == 1
+        assert self.cpu.get_int_value(deadframe, locs, 1) == 10
         assert values == [faildescr, 1, 10]
         assert self.cpu.get_savedata_ref(deadframe)   # not NULL
         assert self.cpu.get_savedata_ref(deadframe) == random_gcref

From noreply at buildbot.pypy.org  Wed Oct 16 23:12:20 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 23:12:20 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: port more stuff
Message-ID: <20131016211220.9DFF81D22DC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67441:cfa5c373b5f6
Date: 2013-10-16 23:09 +0200
http://bitbucket.org/pypy/pypy/changeset/cfa5c373b5f6/

Log:	port more stuff

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2445,8 +2445,10 @@
         def maybe_force(token, flag):
             if flag:
                 deadframe = self.cpu.force(token)
-                values.append(self.cpu.get_int_value(deadframe, 0))
-                values.append(self.cpu.get_int_value(deadframe, 2))
+                fail = self.cpu.get_latest_descr(deadframe)
+                locs = rebuild_locs_from_resumedata(fail)
+                values.append(self.cpu.get_int_value(deadframe, locs, 0))
+                values.append(self.cpu.get_int_value(deadframe, locs, 2))
                 self.cpu.set_savedata_ref(deadframe, random_gcref)
             return 42
 
@@ -2461,28 +2463,34 @@
         i2 = BoxInt()
         tok = BoxPtr()
         faildescr = BasicFailDescr(1)
+        jitcode = JitCode('name')
+        jitcode.setup(num_regs_i=3, num_regs_r=0, num_regs_f=0)
         ops = [
-        ResOperation(rop.FORCE_TOKEN, [], tok),
-        ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], i2,
-                     descr=calldescr),
-        ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
-        ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(0))
+            ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None, descr=jitcode),
+            ResOperation(rop.FORCE_TOKEN, [], tok),
+            ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], i2,
+                         descr=calldescr),
+            ResOperation(rop.RESUME_PUT, [i1, ConstInt(0), ConstInt(0)], None),
+            ResOperation(rop.RESUME_PUT, [i2, ConstInt(0), ConstInt(1)], None),
+            ResOperation(rop.RESUME_PUT, [i0, ConstInt(0), ConstInt(2)], None),
+            ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
+            ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(0))
         ]
-        ops[2].setfailargs([i1, i2, i0])
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, [i0, i1], ops, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 20, 0)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 0
-        assert self.cpu.get_int_value(deadframe, 0) == 42
+        assert self.cpu.get_int_value(deadframe, None, 0) == 42
         assert values == []
 
         deadframe = self.cpu.execute_token(looptoken, 10, 1)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 1
-        assert self.cpu.get_int_value(deadframe, 0) == 1
-        assert self.cpu.get_int_value(deadframe, 1) == 42
-        assert self.cpu.get_int_value(deadframe, 2) == 10
+        locs = rebuild_locs_from_resumedata(fail)
+        assert self.cpu.get_int_value(deadframe, locs, 0) == 1
+        assert self.cpu.get_int_value(deadframe, locs, 1) == 42
+        assert self.cpu.get_int_value(deadframe, locs, 2) == 10
         assert values == [1, 10]
         assert self.cpu.get_savedata_ref(deadframe) == random_gcref
 
@@ -2493,8 +2501,10 @@
         def maybe_force(token, flag):
             if flag:
                 deadframe = self.cpu.force(token)
-                values.append(self.cpu.get_int_value(deadframe, 0))
-                values.append(self.cpu.get_int_value(deadframe, 2))
+                fail = self.cpu.get_latest_descr(deadframe)
+                locs = rebuild_locs_from_resumedata(fail)
+                values.append(self.cpu.get_int_value(deadframe, locs, 0))
+                values.append(self.cpu.get_int_value(deadframe, locs, 1))
                 self.cpu.set_savedata_ref(deadframe, random_gcref)
             return 42.5
 
@@ -2509,30 +2519,36 @@
         f2 = BoxFloat()
         tok = BoxPtr()
         faildescr = BasicFailDescr(1)
+        jitcode = JitCode('name')
+        jitcode.setup(num_regs_i=2, num_regs_r=0, num_regs_f=1)
         ops = [
-        ResOperation(rop.FORCE_TOKEN, [], tok),
-        ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], f2,
-                     descr=calldescr),
-        ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
-        ResOperation(rop.FINISH, [f2], None, descr=BasicFinalDescr(0))
+            ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None, descr=jitcode),
+            ResOperation(rop.FORCE_TOKEN, [], tok),
+            ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], f2,
+                         descr=calldescr),
+            ResOperation(rop.RESUME_PUT, [i1, ConstInt(0), ConstInt(0)], None),
+            ResOperation(rop.RESUME_PUT, [i0, ConstInt(0), ConstInt(1)], None),
+            ResOperation(rop.RESUME_PUT, [f2, ConstInt(0), ConstInt(2)], None),
+            ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
+            ResOperation(rop.FINISH, [f2], None, descr=BasicFinalDescr(0))
         ]
-        ops[2].setfailargs([i1, f2, i0])
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, [i0, i1], ops, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 20, 0)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 0
-        x = self.cpu.get_float_value(deadframe, 0)
+        x = self.cpu.get_float_value(deadframe, None, 0)
         assert longlong.getrealfloat(x) == 42.5
         assert values == []
 
         deadframe = self.cpu.execute_token(looptoken, 10, 1)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 1
-        assert self.cpu.get_int_value(deadframe, 0) == 1
-        x = self.cpu.get_float_value(deadframe, 1)
+        locs = rebuild_locs_from_resumedata(fail)
+        assert self.cpu.get_int_value(deadframe, locs, 0) == 1
+        x = self.cpu.get_float_value(deadframe, locs, 2)
         assert longlong.getrealfloat(x) == 42.5
-        assert self.cpu.get_int_value(deadframe, 2) == 10
+        assert self.cpu.get_int_value(deadframe, locs, 1) == 10
         assert values == [1, 10]
         assert self.cpu.get_savedata_ref(deadframe) == random_gcref
 

From noreply at buildbot.pypy.org  Wed Oct 16 23:12:21 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 23:12:21 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: port more stuff
Message-ID: <20131016211221.DD8F81D22DC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67442:74f3503ae74d
Date: 2013-10-16 23:09 +0200
http://bitbucket.org/pypy/pypy/changeset/74f3503ae74d/

Log:	port more stuff

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2580,7 +2580,7 @@
         deadframe = self.cpu.execute_token(looptoken, ord('G'))
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 0
-        assert self.cpu.get_int_value(deadframe, 0) == ord('g')
+        assert self.cpu.get_int_value(deadframe, None, 0) == ord('g')
 
     def test_call_to_c_function_with_callback(self):
         from rpython.rlib.libffi import CDLL, types, ArgChain, clibffi

From noreply at buildbot.pypy.org  Wed Oct 16 23:12:23 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 23:12:23 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: more tests
Message-ID: <20131016211223.211021D22DC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67443:2cca407bd385
Date: 2013-10-16 23:11 +0200
http://bitbucket.org/pypy/pypy/changeset/2cca407bd385/

Log:	more tests

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2574,7 +2574,6 @@
         ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
         ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(0))
         ]
-        ops[1].setfailargs([i1, i2])
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, [i1], ops, looptoken)
         deadframe = self.cpu.execute_token(looptoken, ord('G'))
@@ -2632,7 +2631,6 @@
         ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
         ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(0))
         ]
-        ops[1].setfailargs([])
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, [i0, i1, i2, i3], ops, looptoken)
         args = [rffi.cast(lltype.Signed, raw),
@@ -2759,7 +2757,6 @@
                 ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
                 ResOperation(rop.FINISH, [b3], None, descr=BasicFinalDescr(0))
                 ]
-            ops[1].setfailargs([])
             looptoken = JitCellToken()
             self.cpu.compile_loop(None, [], ops, looptoken)
 
@@ -2767,7 +2764,7 @@
             fail = self.cpu.get_latest_descr(deadframe)
             assert fail.identifier == 0
             if isinstance(b3, BoxInt):
-                r = self.cpu.get_int_value(deadframe, 0)
+                r = self.cpu.get_int_value(deadframe, None, 0)
                 if isinstance(result, r_singlefloat):
                     assert -sys.maxint-1 <= r <= 0xFFFFFFFF
                     r, = struct.unpack("f", struct.pack("I", r & 0xFFFFFFFF))
@@ -2776,7 +2773,7 @@
                     r = rffi.cast(TP, r)
                 assert r == result
             elif isinstance(b3, BoxFloat):
-                r = self.cpu.get_float_value(deadframe, 0)
+                r = self.cpu.get_float_value(deadframe, None, 0)
                 if isinstance(result, float):
                     r = longlong.getrealfloat(r)
                 else:
@@ -2894,7 +2891,6 @@
                 ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
                 ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(0))
                 ]
-            ops[-2].setfailargs([])
             # keep alive a random subset of the insideboxes
             for b1 in insideboxes:
                 if rnd.random() < keepalive_factor:

From noreply at buildbot.pypy.org  Wed Oct 16 23:45:30 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 16 Oct 2013 23:45:30 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: port tests until I can't any
 more and leave the explanation why the last is
Message-ID: <20131016214530.D64311D22EA@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67444:ae935b78444e
Date: 2013-10-16 23:44 +0200
http://bitbucket.org/pypy/pypy/changeset/ae935b78444e/

Log:	port tests until I can't any more and leave the explanation why the
	last is failing

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2916,18 +2916,22 @@
         i0 = BoxInt()
         i1 = BoxInt()
         faildescr = BasicFailDescr(1)
+        jitcode = JitCode('name')
+        jitcode.setup(num_regs_i=1, num_regs_r=0, num_regs_f=0)
         ops = [
+            ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None, descr=jitcode),
+            ResOperation(rop.RESUME_PUT, [i1, ConstInt(0), ConstInt(0)], None),
             ResOperation(rop.GUARD_NOT_INVALIDATED, [], None, descr=faildescr),
+            ResOperation(rop.LEAVE_FRAME, [], None),
             ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(0))
         ]
-        ops[0].setfailargs([i1])
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, [i0, i1], ops, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, -42, 9)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 0
-        assert self.cpu.get_int_value(deadframe, 0) == -42
+        assert self.cpu.get_int_value(deadframe, None, 0) == -42
         print 'step 1 ok'
         print '-'*79
 
@@ -2936,8 +2940,9 @@
 
         deadframe = self.cpu.execute_token(looptoken, -42, 9)
         fail = self.cpu.get_latest_descr(deadframe)
+        locs = rebuild_locs_from_resumedata(fail)
         assert fail is faildescr
-        assert self.cpu.get_int_value(deadframe, 0) == 9
+        assert self.cpu.get_int_value(deadframe, locs, 0) == 9
         print 'step 2 ok'
         print '-'*79
 
@@ -2948,13 +2953,12 @@
             ResOperation(rop.GUARD_NOT_INVALIDATED, [],None, descr=faildescr2),
             ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(3))
         ]
-        ops[0].setfailargs([])
-        self.cpu.compile_bridge(None, faildescr, [i2], ops, looptoken)
+        self.cpu.compile_bridge(None, faildescr, [i2], locs, ops, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, -42, 9)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 3
-        assert self.cpu.get_int_value(deadframe, 0) == 9
+        assert self.cpu.get_int_value(deadframe, None, 0) == 9
         print 'step 3 ok'
         print '-'*79
 
@@ -2980,7 +2984,6 @@
             ResOperation(rop.LABEL, [i0], None, descr=labeldescr),
             ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(3)),
         ]
-        ops[0].setfailargs([])
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, [i0], ops, looptoken)
         # mark as failing
@@ -2990,12 +2993,12 @@
         ops2 = [
             ResOperation(rop.JUMP, [ConstInt(333)], None, descr=labeldescr),
         ]
-        self.cpu.compile_bridge(None, faildescr, [], ops2, looptoken)
+        self.cpu.compile_bridge(None, faildescr, [], [], ops2, looptoken)
         # run: must not be caught in an infinite loop
         deadframe = self.cpu.execute_token(looptoken, 16)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 3
-        assert self.cpu.get_int_value(deadframe, 0) == 333
+        assert self.cpu.get_int_value(deadframe, None, 0) == 333
 
     # pure do_ / descr features
 
@@ -3168,7 +3171,7 @@
     def test_assembler_call(self):
         called = []
         def assembler_helper(deadframe, virtualizable):
-            assert self.cpu.get_int_value(deadframe, 0) == 97
+            assert self.cpu.get_int_value(deadframe, None, 0) == 97
             called.append(self.cpu.get_latest_descr(deadframe))
             return 4 + 9
 
@@ -3206,12 +3209,12 @@
             EffectInfo.MOST_GENERAL)
         args = [i+1 for i in range(10)]
         deadframe = self.cpu.execute_token(looptoken, *args)
-        assert self.cpu.get_int_value(deadframe, 0) == 55
+        assert self.cpu.get_int_value(deadframe, None, 0) == 55
         ops = '''
         [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9]
         i10 = int_add(i0, 42)
         i11 = call_assembler(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken)
-        guard_not_forced()[]
+        guard_not_forced()
         finish(i11)
         '''
         loop = parse(ops, namespace=locals())
@@ -3219,7 +3222,7 @@
         self.cpu.compile_loop(None, loop.inputargs, loop.operations, othertoken)
         args = [i+1 for i in range(10)]
         deadframe = self.cpu.execute_token(othertoken, *args)
-        assert self.cpu.get_int_value(deadframe, 0) == 13
+        assert self.cpu.get_int_value(deadframe, None, 0) == 13
         assert called == [finish_descr]
 
         # test the fast path, which should not call assembler_helper()
@@ -3229,7 +3232,7 @@
         self.cpu.compile_loop(None, loop.inputargs, loop.operations, othertoken)
         args = [i+1 for i in range(10)]
         deadframe = self.cpu.execute_token(othertoken, *args)
-        assert self.cpu.get_int_value(deadframe, 0) == 97
+        assert self.cpu.get_int_value(deadframe, None, 0) == 97
         assert not called
 
     def test_assembler_call_propagate_exc(self):
@@ -3273,21 +3276,21 @@
         ops = '''
         [i0]
         i11 = call_assembler(i0, descr=looptoken)
-        guard_not_forced()[]
+        guard_not_forced()
         finish(i11)
         '''
         loop = parse(ops, namespace=locals())
         othertoken = JitCellToken()
         self.cpu.compile_loop(None, loop.inputargs, loop.operations, othertoken)
         deadframe = self.cpu.execute_token(othertoken, sys.maxint - 1)
-        assert self.cpu.get_int_value(deadframe, 0) == 3
+        assert self.cpu.get_int_value(deadframe, None, 0) == 3
 
     def test_assembler_call_float(self):
         if not self.cpu.supports_floats:
             py.test.skip("requires floats")
         called = []
         def assembler_helper(deadframe, virtualizable):
-            x = self.cpu.get_float_value(deadframe, 0)
+            x = self.cpu.get_float_value(deadframe, None, 0)
             assert longlong.getrealfloat(x) == 1.2 + 3.2
             called.append(self.cpu.get_latest_descr(deadframe))
             print '!' * 30 + 'assembler_helper'
@@ -3320,12 +3323,12 @@
         args = [longlong.getfloatstorage(1.2),
                 longlong.getfloatstorage(2.3)]
         deadframe = self.cpu.execute_token(looptoken, *args)
-        x = self.cpu.get_float_value(deadframe, 0)
+        x = self.cpu.get_float_value(deadframe, None, 0)
         assert longlong.getrealfloat(x) == 1.2 + 2.3
         ops = '''
         [f4, f5]
         f3 = call_assembler(f4, f5, descr=looptoken)
-        guard_not_forced()[]
+        guard_not_forced()
         finish(f3)
         '''
         loop = parse(ops, namespace=locals())
@@ -3334,7 +3337,7 @@
         args = [longlong.getfloatstorage(1.2),
                 longlong.getfloatstorage(3.2)]
         deadframe = self.cpu.execute_token(othertoken, *args)
-        x = self.cpu.get_float_value(deadframe, 0)
+        x = self.cpu.get_float_value(deadframe, None, 0)
         assert longlong.getrealfloat(x) == 13.5
         assert called == [finish_descr]
 
@@ -3346,7 +3349,7 @@
         args = [longlong.getfloatstorage(1.2),
                 longlong.getfloatstorage(4.2)]
         deadframe = self.cpu.execute_token(othertoken, *args)
-        x = self.cpu.get_float_value(deadframe, 0)
+        x = self.cpu.get_float_value(deadframe, None, 0)
         assert longlong.getrealfloat(x) == 1.2 + 4.2
         assert not called
 
@@ -3379,7 +3382,7 @@
             py.test.skip("requires floats")
         called = []
         def assembler_helper(deadframe, virtualizable):
-            x = self.cpu.get_float_value(deadframe, 0)
+            x = self.cpu.get_float_value(deadframe, None, 0)
             assert longlong.getrealfloat(x) == 1.25 + 3.25
             called.append(self.cpu.get_latest_descr(deadframe))
             return 13.5
@@ -3410,14 +3413,14 @@
         args = [longlong.getfloatstorage(1.25),
                 longlong.getfloatstorage(2.35)]
         deadframe = self.cpu.execute_token(looptoken, *args)
-        x = self.cpu.get_float_value(deadframe, 0)
+        x = self.cpu.get_float_value(deadframe, None, 0)
         assert longlong.getrealfloat(x) == 1.25 + 2.35
         assert not called
 
         ops = '''
         [f4, f5]
         f3 = call_assembler(f4, f5, descr=looptoken)
-        guard_not_forced()[]
+        guard_not_forced()
         finish(f3)
         '''
         loop = parse(ops, namespace=locals())
@@ -3428,7 +3431,7 @@
         args = [longlong.getfloatstorage(1.25),
                 longlong.getfloatstorage(3.25)]
         deadframe = self.cpu.execute_token(othertoken, *args)
-        x = self.cpu.get_float_value(deadframe, 0)
+        x = self.cpu.get_float_value(deadframe, None, 0)
         assert longlong.getrealfloat(x) == 13.5
         assert called == [finish_descr]
         del called[:]
@@ -3451,7 +3454,7 @@
         args = [longlong.getfloatstorage(6.0),
                 longlong.getfloatstorage(1.5)]         # 6.0-1.5 == 1.25+3.25
         deadframe = self.cpu.execute_token(othertoken, *args)
-        x = self.cpu.get_float_value(deadframe, 0)
+        x = self.cpu.get_float_value(deadframe, None, 0)
         assert longlong.getrealfloat(x) == 13.5
         assert called == [finish_descr2]
 
@@ -3840,10 +3843,14 @@
         targettoken1 = TargetToken()
         targettoken2 = TargetToken()
         faildescr = BasicFailDescr(2)
+        jitcode = JitCode('name')
+        jitcode.setup(num_regs_i=1, num_regs_r=0, num_regs_f=0)
         operations = [
+            ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None, descr=jitcode),
             ResOperation(rop.LABEL, [i0], None, descr=targettoken1),
             ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
             ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2),
+            ResOperation(rop.RESUME_PUT, [i1, ConstInt(0), ConstInt(0)], None),
             ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr),
             ResOperation(rop.LABEL, [i1], None, descr=targettoken2),
             ResOperation(rop.INT_GE, [i1, ConstInt(0)], i3),
@@ -3851,14 +3858,13 @@
             ResOperation(rop.JUMP, [i1], None, descr=targettoken1),
             ]
         inputargs = [i0]
-        operations[3].setfailargs([i1])
-        operations[6].setfailargs([i1])
 
         self.cpu.compile_loop(None, inputargs, operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
+        locs = rebuild_locs_from_resumedata(fail)
         assert fail.identifier == 2
-        res = self.cpu.get_int_value(deadframe, 0)
+        res = self.cpu.get_int_value(deadframe, locs, 0)
         assert res == 10
 
         inputargs2 = [i0]
@@ -3866,12 +3872,13 @@
             ResOperation(rop.INT_SUB, [i0, ConstInt(20)], i2),
             ResOperation(rop.JUMP, [i2], None, descr=targettoken2),
             ]
-        self.cpu.compile_bridge(None, faildescr, inputargs2, operations2, looptoken)
+        self.cpu.compile_bridge(None, faildescr, inputargs2, locs, operations2, looptoken)
 
         deadframe = self.cpu.execute_token(looptoken, 2)
         fail = self.cpu.get_latest_descr(deadframe)
+        locs = rebuild_locs_from_resumedata(fail)
         assert fail.identifier == 3
-        res = self.cpu.get_int_value(deadframe, 0)
+        res = self.cpu.get_int_value(deadframe, locs, 0)
         assert res == -10
 
     def test_int_force_ge_zero(self):
@@ -3886,7 +3893,7 @@
         self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken)
         for inp, outp in [(2,2), (-3, 0)]:
             deadframe = self.cpu.execute_token(looptoken, inp)
-            assert outp == self.cpu.get_int_value(deadframe, 0)
+            assert outp == self.cpu.get_int_value(deadframe, None, 0)
 
     def test_compile_asmlen(self):
         from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU
@@ -3894,13 +3901,17 @@
             py.test.skip("pointless test on non-asm")
         from rpython.jit.backend.tool.viewcode import machine_code_dump
         import ctypes
+        jitcode = JitCode('name')
+        jitcode.setup(num_regs_i=1, num_regs_r=0, num_regs_f=0)
         targettoken = TargetToken()
         ops = """
         [i2]
+        enter_frame(-1, descr=jitcode)
         i0 = same_as(i2)    # but forced to be in a register
         label(i0, descr=targettoken)
         i1 = int_add(i0, i0)
-        guard_true(i1, descr=faildescr) [i1]
+        resume_put(i1, 0, 0)
+        guard_true(i1, descr=faildescr)
         jump(i1, descr=targettoken)
         """
         faildescr = BasicFailDescr(2)
@@ -3913,8 +3924,9 @@
         looptoken = JitCellToken()
         self.cpu.assembler.set_debug(False)
         info = self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken)
+        locs = rebuild_locs_from_resumedata(faildescr)
         bridge_info = self.cpu.compile_bridge(None, faildescr, bridge.inputargs,
-                                              bridge.operations,
+                                              locs, bridge.operations,
                                               looptoken)
         self.cpu.assembler.set_debug(True) # always on untranslated
         assert info.asmlen != 0
@@ -3951,12 +3963,16 @@
         targettoken1 = TargetToken()
         faildescr1 = BasicFailDescr(2)
         inputargs = [i0]
+        jitcode = JitCode('name')
+        jitcode.setup(num_regs_i=1, num_regs_r=0, num_regs_f=0)
         operations = [
+            ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None, descr=jitcode),
             ResOperation(rop.INT_LE, [i0, ConstInt(1)], i1),
+            ResOperation(rop.RESUME_PUT, [i1, ConstInt(0), ConstInt(0)], None),
             ResOperation(rop.GUARD_TRUE, [i1], None, descr=faildescr1),
+            ResOperation(rop.LEAVE_FRAME, [], None),
             ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(1234)),
             ]
-        operations[1].setfailargs([i0])
         self.cpu.compile_loop(None, inputargs, operations, looptoken1)
 
         def func(a, b, c, d, e, f, g, h, i):
@@ -4010,8 +4026,8 @@
             ResOperation(rop.GUARD_TRUE, [i20], None, descr=BasicFailDescr(42)),
             ResOperation(rop.JUMP, [i19], None, descr=targettoken1),
             ]
-        operations2[-2].setfailargs([])
-        self.cpu.compile_bridge(None, faildescr1, inputargs, operations2, looptoken1)
+        locs = rebuild_locs_from_resumedata(faildescr1)
+        self.cpu.compile_bridge(None, faildescr1, inputargs, locs, operations2, looptoken1)
 
         looptoken2 = JitCellToken()
         inputargs = [BoxInt()]
@@ -4032,14 +4048,13 @@
             ResOperation(rop.GUARD_NONNULL_CLASS, [t_box, T_box], None,
                                                         descr=faildescr),
             ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(1))]
-        operations[0].setfailargs([])
         looptoken = JitCellToken()
         inputargs = [t_box]
         self.cpu.compile_loop(None, inputargs, operations, looptoken)
         operations = [
             ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(99))
         ]
-        self.cpu.compile_bridge(None, faildescr, [], operations, looptoken)
+        self.cpu.compile_bridge(None, faildescr, [], [], operations, looptoken)
         deadframe = self.cpu.execute_token(looptoken, null_box.getref_base())
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 99
@@ -4070,7 +4085,7 @@
             self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken)
             deadframe = self.cpu.execute_token(looptoken,
                                                rffi.cast(lltype.Signed, p), 16)
-            result = self.cpu.get_int_value(deadframe, 0)
+            result = self.cpu.get_int_value(deadframe, None, 0)
             assert result == rffi.cast(lltype.Signed, value)
             rawstorage.free_raw_storage(p)
 
@@ -4100,7 +4115,7 @@
             self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken)
             deadframe = self.cpu.execute_token(looptoken,
                                                rffi.cast(lltype.Signed, p), 16)
-            result = self.cpu.get_float_value(deadframe, 0)
+            result = self.cpu.get_float_value(deadframe, None, 0)
             result = longlong.getrealfloat(result)
             assert result == rffi.cast(lltype.Float, value)
             rawstorage.free_raw_storage(p)
@@ -4130,7 +4145,7 @@
             self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken)
             deadframe = self.cpu.execute_token(looptoken,
                                                rffi.cast(lltype.Signed, p), 16)
-            result = self.cpu.get_int_value(deadframe, 0)
+            result = self.cpu.get_int_value(deadframe, None, 0)
             assert result == longlong.singlefloat2int(value)
             rawstorage.free_raw_storage(p)
 
@@ -4238,7 +4253,9 @@
         values = []
         def maybe_force(token, flag):
             deadframe = self.cpu.force(token)
-            values.append(self.cpu.get_int_value(deadframe, 0))
+            fail = self.cpu.get_latest_descr(deadframe)
+            locs = rebuild_locs_from_resumedata(fail)
+            values.append(self.cpu.get_int_value(deadframe, locs, 0))
             return 42
 
         FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Signed)
@@ -4251,26 +4268,36 @@
         i2 = BoxInt()
         tok = BoxPtr()
         faildescr = BasicFailDescr(23)
+        jitcode = JitCode('name')
+        jitcode.setup(num_regs_i=1, num_regs_r=0, num_regs_f=0)
         ops = [
+        ResOperation(rop.ENTER_FRAME, [ConstInt(-1)], None, descr=jitcode),
         ResOperation(rop.FORCE_TOKEN, [], tok),
         ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], i2,
                      descr=calldescr),
+        ResOperation(rop.RESUME_PUT, [i2, ConstInt(0), ConstInt(0)], None),
         ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr),
         ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(0))
         ]
-        ops[2].setfailargs([i2])
         looptoken = JitCellToken()
         self.cpu.compile_loop(None, [i0, i1], ops, looptoken)
         deadframe = self.cpu.execute_token(looptoken, 20, 0)
         fail = self.cpu.get_latest_descr(deadframe)
         assert fail.identifier == 23
-        assert self.cpu.get_int_value(deadframe, 0) == 42
+        locs = rebuild_locs_from_resumedata(fail)
+        assert self.cpu.get_int_value(deadframe, locs, 0) == 42
         # make sure that force reads the registers from a zeroed piece of
         # memory
         assert values[0] == 0
 
     def test_compile_bridge_while_running(self):
+        XXX # it crashes because the regalloc does not inherit liveness
+        # rules from the parent, while it shoul
+
         def func():
+            jitcode2 = JitCode('name2')
+            jitcode2.setup(num_regs_i=7, num_regs_r=0, num_regs_f=0)
+
             bridge = parse("""
             [i1, i2, px]
             i3 = int_add(i1, i2)
@@ -4289,12 +4316,24 @@
             force_spill(i7)
             force_spill(i8)
             force_spill(i9)
+            enter_frame(1, descr=jitcode2)
             call(ConstClass(func2_ptr), 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, descr=calldescr2)
-            guard_true(i1, descr=guarddescr) [i1, i2, i3, i4, i5, i6, i7, i8, i9, px]
+            resume_put(i3, 0, 0)
+            resume_put(i4, 0, 1)
+            resume_put(i5, 0, 2)
+            resume_put(i6, 0, 3)
+            resume_put(i7, 0, 4)
+            resume_put(i8, 0, 5)
+            resume_put(i9, 0, 6)
+            guard_true(i1, descr=guarddescr)
+            leave_frame()
+            leave_frame()
             finish(i1, descr=finaldescr)
             """, namespace={'finaldescr': finaldescr, 'calldescr2': calldescr2,
-                            'guarddescr': guarddescr, 'func2_ptr': func2_ptr})
-            self.cpu.compile_bridge(None, faildescr, bridge.inputargs,
+                            'guarddescr': guarddescr, 'func2_ptr': func2_ptr,
+                            'jitcode2': jitcode2})
+            locs = rebuild_locs_from_resumedata(faildescr)
+            self.cpu.compile_bridge(None, faildescr, bridge.inputargs, locs,
                                     bridge.operations, looptoken)
 
         cpu = self.cpu
@@ -4320,11 +4359,18 @@
         faildescr = BasicFailDescr(0)
 
         looptoken = JitCellToken()
+        jitcode = JitCode('name')
+        jitcode.setup(num_regs_i=2, num_regs_r=1, num_regs_f=0)
         loop = parse("""
         [i0, i1, i2]
+        enter_frame(-1, descr=jitcode)
         call(ConstClass(func_ptr), descr=calldescr)
         px = force_token()
-        guard_true(i0, descr=faildescr) [i1, i2, px]
+        resume_put(i1, 0, 0)
+        resume_put(i2, 0, 1)
+        resume_put(px, 0, 2)
+        guard_true(i0, descr=faildescr)
+        leave_frame()
         finish(i2, descr=finaldescr2)
         """, namespace=locals())
         self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken)
@@ -4337,7 +4383,8 @@
 
         frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame)
         assert len(frame.jf_frame) == frame.jf_frame_info.jfi_frame_depth
-        ref = self.cpu.get_ref_value(frame, 9)
+        locs = rebuild_locs_from_resumedata(guarddescr)
+        ref = self.cpu.get_ref_value(frame, locs, 2)
         token = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, ref)
         assert token != frame
         token = token.resolve()

From noreply at buildbot.pypy.org  Wed Oct 16 23:46:00 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 23:46:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: test/fix another ndarray exception
Message-ID: <20131016214600.52E8E1D22EA@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67445:04050eb4a2d9
Date: 2013-10-16 16:51 -0400
http://bitbucket.org/pypy/pypy/changeset/04050eb4a2d9/

Log:	test/fix another ndarray exception

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -380,8 +380,8 @@
 
 class NonWritableArray(ConcreteArray):
     def descr_setitem(self, space, orig_array, w_index, w_value):
-        raise OperationError(space.w_RuntimeError, space.wrap(
-            "array is not writable"))
+        raise OperationError(space.w_ValueError, space.wrap(
+            "assignment destination is read-only"))
 
 
 class SliceArray(BaseConcreteArray):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1442,7 +1442,7 @@
         assert a[5] == 50
         b = a.imag
         assert b[7] == 0
-        raises(RuntimeError, 'b[7] = -2')
+        raises(ValueError, 'b[7] = -2')
         raises(TypeError, 'a.imag = -2')
         a = array(['abc','def'],dtype='S3')
         b = a.real

From noreply at buildbot.pypy.org  Wed Oct 16 23:46:01 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 23:46:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix imag() on flexible arrays to match
	numpy
Message-ID: <20131016214601.7EA091D22EA@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67446:3f72f9113817
Date: 2013-10-16 16:56 -0400
http://bitbucket.org/pypy/pypy/changeset/3f72f9113817/

Log:	fix imag() on flexible arrays to match numpy

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -102,13 +102,10 @@
             dtype =  self.dtype.float_type
             return SliceArray(self.start + dtype.get_size(), strides,
                     backstrides, self.get_shape(), self, orig_array, dtype=dtype)
-        if self.dtype.is_flexible_type():
-            # numpy returns self for self.imag
-            return SliceArray(self.start, strides, backstrides,
-                    self.get_shape(), self, orig_array)
         impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides,
                              backstrides)
-        impl.fill(self.dtype.box(0))
+        if not self.dtype.is_flexible_type():
+            impl.fill(self.dtype.box(0))
         return impl
 
     def set_imag(self, space, orig_array, w_value):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1450,7 +1450,7 @@
         assert a[1] == b[1]
         b[1] = 'xyz'
         assert a[1] == 'xyz'
-        assert a.imag[0] == 'abc'
+        assert a.imag[0] == ''
         raises(TypeError, 'a.imag = "qop"')
         a=array([[1+1j, 2-3j, 4+5j],[-6+7j, 8-9j, -2-1j]])
         assert a.real[0,1] == 2

From noreply at buildbot.pypy.org  Wed Oct 16 23:46:02 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 23:46:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix ndarray.put() exception
Message-ID: <20131016214602.B872B1D22EA@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67447:47108b081744
Date: 2013-10-16 17:08 -0400
http://bitbucket.org/pypy/pypy/changeset/47108b081744/

Log:	fix ndarray.put() exception

diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -229,8 +229,8 @@
 
         if index < 0 or index >= arr.get_size():
             if constants.MODES[mode] == constants.MODE_RAISE:
-                raise OperationError(space.w_ValueError, space.wrap(
-                    "invalid entry in choice array"))
+                raise OperationError(space.w_IndexError, space.wrap(
+                    "index %d is out of bounds for axis 0 with size %d" % (index, arr.get_size())))
             elif constants.MODES[mode] == constants.MODE_WRAP:
                 index = index % arr.get_size()
             else:
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
--- a/pypy/module/micronumpy/test/test_arrayops.py
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -153,5 +153,5 @@
         a = arange(5)
         a.put(22, -5, mode='wrap')
         assert (a == array([0, 1, -5, 3, 4])).all()
-        raises(ValueError, "arange(5).put(22, -5, mode='raise')")
+        raises(IndexError, "arange(5).put(22, -5, mode='raise')")
         raises(ValueError, "arange(5).put(22, -5, mode='wrongmode')")

From noreply at buildbot.pypy.org  Wed Oct 16 23:46:04 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 23:46:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: cleanup these error messages
Message-ID: <20131016214604.029311D22EA@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67448:2fc0e3aaa45a
Date: 2013-10-16 17:44 -0400
http://bitbucket.org/pypy/pypy/changeset/2fc0e3aaa45a/

Log:	cleanup these error messages

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -126,7 +126,8 @@
                 idx = self.get_shape()[i] + idx
             if idx < 0 or idx >= self.get_shape()[i]:
                 raise operationerrfmt(space.w_IndexError,
-                      "index (%d) out of range (0<=index<%d", i, self.get_shape()[i],
+                      "index %d is out of bounds for axis %d with size %d",
+                      idx, i, self.get_shape()[i],
                 )
             item += idx * strides[i]
         return item
@@ -142,7 +143,8 @@
                 idx = shape[i] + idx
             if idx < 0 or idx >= shape[i]:
                 raise operationerrfmt(space.w_IndexError,
-                      "index (%d) out of range (0<=index<%d", i, shape[i],
+                      "index %d is out of bounds for axis %d with size %d",
+                      idx, i, self.get_shape()[i],
                 )
             item += idx * strides[i]
         return item

From noreply at buildbot.pypy.org  Wed Oct 16 23:46:05 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 16 Oct 2013 23:46:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: enhance ndarray mode argument parsing,
	test
Message-ID: <20131016214605.334861D22EA@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67449:3388d5c537f8
Date: 2013-10-16 17:42 -0400
http://bitbucket.org/pypy/pypy/changeset/3388d5c537f8/

Log:	enhance ndarray mode argument parsing, test

diff --git a/pypy/module/micronumpy/constants.py b/pypy/module/micronumpy/constants.py
--- a/pypy/module/micronumpy/constants.py
+++ b/pypy/module/micronumpy/constants.py
@@ -1,4 +1,21 @@
+from pypy.interpreter.error import OperationError
 
-MODE_WRAP, MODE_RAISE, MODE_CLIP = range(3)
+MODE_CLIP, MODE_WRAP, MODE_RAISE = range(3)
 
-MODES = {'wrap': MODE_WRAP, 'raise': MODE_RAISE, 'clip': MODE_CLIP}
+def clipmode_converter(space, w_mode):
+    if space.is_none(w_mode):
+        return MODE_RAISE
+    if space.isinstance_w(w_mode, space.w_str):
+        mode = space.str_w(w_mode)
+        if mode.startswith('C') or mode.startswith('c'):
+            return MODE_CLIP
+        if mode.startswith('W') or mode.startswith('w'):
+            return MODE_WRAP
+        if mode.startswith('R') or mode.startswith('r'):
+            return MODE_RAISE
+    elif space.isinstance_w(w_mode, space.w_int):
+        mode = space.int_w(w_mode)
+        if MODE_CLIP <= mode <= MODE_RAISE:
+            return mode
+    raise OperationError(space.w_TypeError,
+                         space.wrap("clipmode not understood"))
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -1,10 +1,9 @@
-
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
-from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs
+from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs, constants
 from pypy.module.micronumpy.iter import Chunk, Chunks
 from pypy.module.micronumpy.strides import shape_agreement,\
      shape_agreement_multiple
-from pypy.module.micronumpy.constants import MODES
+from pypy.module.micronumpy.constants import clipmode_converter
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 
@@ -171,8 +170,7 @@
 def count_nonzero(space, w_obj):
     return space.wrap(loop.count_all_true(convert_to_array(space, w_obj)))
 
- at unwrap_spec(mode=str)
-def choose(space, w_arr, w_choices, w_out, mode):
+def choose(space, w_arr, w_choices, w_out, w_mode):
     arr = convert_to_array(space, w_arr)
     choices = [convert_to_array(space, w_item) for w_item
                in space.listview(w_choices)]
@@ -187,23 +185,16 @@
     shape = shape_agreement_multiple(space, choices + [w_out])
     out = interp_dtype.dtype_agreement(space, choices, shape, w_out)
     dtype = out.get_dtype()
-    if mode not in MODES:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("mode %s not known" % (mode,)))
-    loop.choose(space, arr, choices, shape, dtype, out, MODES[mode])
+    mode = clipmode_converter(space, w_mode)
+    loop.choose(space, arr, choices, shape, dtype, out, mode)
     return out
 
-
- at unwrap_spec(mode=str)
-def put(space, w_arr, w_indices, w_values, mode='raise'):
-    from pypy.module.micronumpy import constants
+def put(space, w_arr, w_indices, w_values, w_mode):
     from pypy.module.micronumpy.support import int_w
 
     arr = convert_to_array(space, w_arr)
+    mode = clipmode_converter(space, w_mode)
 
-    if mode not in constants.MODES:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("mode %s not known" % (mode,)))
     if not w_indices:
         raise OperationError(space.w_ValueError,
                              space.wrap("indice list cannot be empty"))
@@ -228,13 +219,13 @@
         index = int_w(space, idx)
 
         if index < 0 or index >= arr.get_size():
-            if constants.MODES[mode] == constants.MODE_RAISE:
+            if mode == constants.MODE_RAISE:
                 raise OperationError(space.w_IndexError, space.wrap(
                     "index %d is out of bounds for axis 0 with size %d" % (index, arr.get_size())))
-            elif constants.MODES[mode] == constants.MODE_WRAP:
+            elif mode == constants.MODE_WRAP:
                 index = index % arr.get_size()
             else:
-                assert constants.MODES[mode] == constants.MODE_CLIP
+                assert mode == constants.MODE_CLIP
                 if index < 0:
                     index = 0
                 else:
@@ -247,7 +238,6 @@
 
         arr.setitem(space, [index], dtype.coerce(space, value))
 
-
 def diagonal(space, arr, offset, axis1, axis2):
     shape = arr.get_shape()
     shapelen = len(shape)
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,4 +1,3 @@
-
 from pypy.interpreter.error import operationerrfmt, OperationError
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -14,7 +13,7 @@
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy import loop
 from pypy.module.micronumpy.dot import match_dot_shapes
-from pypy.module.micronumpy.interp_arrayops import repeat, choose
+from pypy.module.micronumpy.interp_arrayops import repeat, choose, put
 from pypy.module.micronumpy.arrayimpl import scalar
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rlib import jit
@@ -509,9 +508,8 @@
             loop.byteswap(self.implementation, w_res.implementation)
             return w_res
 
-    @unwrap_spec(mode=str)
-    def descr_choose(self, space, w_choices, w_out=None, mode='raise'):
-        return choose(space, self, w_choices, w_out, mode)
+    def descr_choose(self, space, w_choices, w_out=None, w_mode=None):
+        return choose(space, self, w_choices, w_out, w_mode)
 
     def descr_clip(self, space, w_min, w_max, w_out=None):
         if space.is_none(w_out):
@@ -590,10 +588,8 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "ptp (peak to peak) not implemented yet"))
 
-    @unwrap_spec(mode=str)
-    def descr_put(self, space, w_indices, w_values, mode='raise'):
-        from pypy.module.micronumpy.interp_arrayops import put
-        put(space, self, w_indices, w_values, mode)
+    def descr_put(self, space, w_indices, w_values, w_mode=None):
+        put(space, self, w_indices, w_values, w_mode)
 
     def descr_resize(self, space, w_new_shape, w_refcheck=True):
         raise OperationError(space.w_NotImplementedError, space.wrap(
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
--- a/pypy/module/micronumpy/test/test_arrayops.py
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -154,4 +154,11 @@
         a.put(22, -5, mode='wrap')
         assert (a == array([0, 1, -5, 3, 4])).all()
         raises(IndexError, "arange(5).put(22, -5, mode='raise')")
-        raises(ValueError, "arange(5).put(22, -5, mode='wrongmode')")
+        raises(IndexError, "arange(5).put(22, -5, mode=2)")  # raise
+        a.put(22, -10, mode='wrongmode_starts_with_w_so_wrap')
+        assert (a == array([0, 1, -10, 3, 4])).all()
+        a.put(22, -15, mode='cccccccc')
+        assert (a == array([0, 1, -10, 3, -15])).all()
+        a.put(23, -1, mode=1)  # wrap
+        assert (a == array([0, 1, -10, -1, -15])).all()
+        raises(TypeError, "arange(5).put(22, -5, mode='zzzz')")  # unrecognized mode

From noreply at buildbot.pypy.org  Thu Oct 17 00:29:25 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 00:29:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix scalar any/all return types
Message-ID: <20131016222925.23A1C1D22DC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67450:11fec059468c
Date: 2013-10-16 18:01 -0400
http://bitbucket.org/pypy/pypy/changeset/11fec059468c/

Log:	fix scalar any/all return types

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -227,11 +227,11 @@
 
     def descr_any(self, space):
         value = space.is_true(self)
-        return space.wrap(W_BoolBox(value))
+        return self.get_dtype(space).box(value)
 
     def descr_all(self, space):
         value = space.is_true(self)
-        return space.wrap(W_BoolBox(value))
+        return self.get_dtype(space).box(value)
 
     def descr_ravel(self, space):
         from pypy.module.micronumpy.base import convert_to_array
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -745,11 +745,11 @@
         x = numpy.bool_(False)
         assert not x.any()
         assert not x.all()
-        #
+        assert isinstance(x.any(), numpy.bool_)
         x = numpy.float64(0)
         assert not x.any()
         assert not x.all()
-        assert isinstance(x.any(), numpy.bool_)
+        assert isinstance(x.any(), numpy.float64)
 
     def test_ravel(self):
         from numpypy import float64, int8, array

From noreply at buildbot.pypy.org  Thu Oct 17 01:46:40 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 01:46:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: provide bool() for complex types, test
Message-ID: <20131016234640.789F51D22DC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67451:271637e39b8a
Date: 2013-10-16 19:41 -0400
http://bitbucket.org/pypy/pypy/changeset/271637e39b8a/

Log:	provide bool() for complex types, test

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -737,19 +737,38 @@
         assert dtype('i4').isnative == True
         assert dtype('>i8').isnative == False
 
-    def test_any_all(self):
+    def test_any_all_nonzero(self):
         import numpypy as numpy
         x = numpy.bool_(True)
         assert x.any()
         assert x.all()
+        assert x.__nonzero__()
+        assert isinstance(x.any(), numpy.bool_)
+        assert isinstance(x.__nonzero__(), bool)
         x = numpy.bool_(False)
         assert not x.any()
         assert not x.all()
+        assert not x.__nonzero__()
         assert isinstance(x.any(), numpy.bool_)
+        assert isinstance(x.__nonzero__(), bool)
         x = numpy.float64(0)
         assert not x.any()
         assert not x.all()
+        assert not x.__nonzero__()
         assert isinstance(x.any(), numpy.float64)
+        assert isinstance(x.__nonzero__(), bool)
+        x = numpy.complex128(0)
+        assert not x.any()
+        assert not x.all()
+        assert not x.__nonzero__()
+        assert isinstance(x.any(), numpy.complex128)
+        assert isinstance(x.__nonzero__(), bool)
+        x = numpy.complex128(0+1j)
+        assert x.any()
+        assert x.all()
+        assert x.__nonzero__()
+        assert isinstance(x.any(), numpy.complex128)
+        assert isinstance(x.__nonzero__(), bool)
 
     def test_ravel(self):
         from numpypy import float64, int8, array
@@ -762,7 +781,6 @@
         assert (x == array(42)).all()
 
 
-
 class AppTestStrUnicodeDtypes(BaseNumpyAppTest):
     def test_str_unicode(self):
         skip('numpypy differs from numpy')
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1148,9 +1148,13 @@
         return v
 
     def to_builtin_type(self, space, box):
-        real,imag = self.for_computation(self.unbox(box))
+        real, imag = self.for_computation(self.unbox(box))
         return space.newcomplex(real, imag)
 
+    def bool(self, v):
+        real, imag = self.for_computation(self.unbox(v))
+        return bool(real) or bool(imag)
+
     def read_bool(self, arr, i, offset):
         v = self.for_computation(self._read(arr.storage, i, offset))
         return bool(v[0]) or bool(v[1])

From noreply at buildbot.pypy.org  Thu Oct 17 02:21:45 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 02:21:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix broken imports in gc test_direct
Message-ID: <20131017002145.20D1D1D22E7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67452:94270f975cb6
Date: 2013-10-16 20:21 -0400
http://bitbucket.org/pypy/pypy/changeset/94270f975cb6/

Log:	fix broken imports in gc test_direct

diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -10,7 +10,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.memory.gctypelayout import TypeLayoutBuilder
 from rpython.rlib.rarithmetic import LONG_BIT, is_valid_int
-from rpython.memory.gc import incminimark
+from rpython.memory.gc import minimark, incminimark
 
 WORD = LONG_BIT // 8
 
@@ -591,8 +591,6 @@
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
 
     def test_write_barrier_marking_simple(self):
-        from rpython.memory.gc import incminimark
-
         for i in range(2):
             curobj = self.malloc(S)
             curobj.x = i

From noreply at buildbot.pypy.org  Thu Oct 17 07:17:40 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 07:17:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix reciprocal of numpy int(0)
Message-ID: <20131017051740.53E471C00BB@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67453:7c0362505295
Date: 2013-10-17 01:16 -0400
http://bitbucket.org/pypy/pypy/changeset/7c0362505295/

Log:	fix reciprocal of numpy int(0)

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -323,11 +323,10 @@
             reference = [0, -1, 0, 1, 0]
             if dtype[0] == 'u':
                 reference[1] = 0
-            # XXX need to fix specialization issue in types.py first
-            #elif dtype == 'int32':
-            #        reference[2] = -2147483648
-            #elif dtype == 'int64':
-            #        reference[2] = -9223372036854775808
+            elif dtype == 'int32':
+                    reference[2] = -2147483648
+            elif dtype == 'int64':
+                    reference[2] = -9223372036854775808
             a = array([-2, -1, 0, 1, 2], dtype)
             b = reciprocal(a)
             assert (b == reference).all()
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -12,7 +12,7 @@
 from rpython.rlib.rawstorage import (alloc_raw_storage, raw_storage_setitem,
                                   raw_storage_getitem)
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import widen, byteswap, r_ulonglong
+from rpython.rlib.rarithmetic import widen, byteswap, r_ulonglong, most_neg_value_of
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rstruct.runpack import runpack
 from rpython.rlib.rstruct.nativefmttable import native_is_bigendian
@@ -521,18 +521,17 @@
     def invert(self, v):
         return ~v
 
-    @simple_unary_op
+    @specialize.argtype(1)
     def reciprocal(self, v):
-        if v == 0:
+        raw = self.for_computation(self.unbox(v))
+        ans = 0
+        if raw == 0:
             # XXX good place to warn
-            # XXX can't do the following, func is specialized only on argtype(v)
-            # (which is the same for all int classes)
-            #if self.T in (rffi.INT, rffi.LONG):
-            #    return most_neg_value_of(self.T)
-            return 0
-        if abs(v) == 1:
-            return v
-        return 0
+            if self.T is rffi.INT or self.T is rffi.LONG:
+                ans = most_neg_value_of(self.T)
+        elif abs(raw) == 1:
+            ans = raw
+        return self.box(ans)
 
     @specialize.argtype(1)
     def round(self, v, decimals=0):

From noreply at buildbot.pypy.org  Thu Oct 17 09:13:57 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 09:13:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: clean up ufuncs, fix some behaviors
Message-ID: <20131017071357.224201C00BB@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67454:022ea29b5d2e
Date: 2013-10-17 03:10 -0400
http://bitbucket.org/pypy/pypy/changeset/022ea29b5d2e/

Log:	clean up ufuncs, fix some behaviors

diff --git a/lib_pypy/numpypy/lib/__init__.py b/lib_pypy/numpypy/lib/__init__.py
--- a/lib_pypy/numpypy/lib/__init__.py
+++ b/lib_pypy/numpypy/lib/__init__.py
@@ -5,10 +5,12 @@
 from .function_base import *
 from .shape_base import *
 from .twodim_base import *
+from .ufunclike import *
 from .utils import *
 
 __all__ = ['math']
 __all__ += function_base.__all__
 __all__ += shape_base.__all__
 __all__ += twodim_base.__all__
+__all__ += ufunclike.__all__
 __all__ += utils.__all__
diff --git a/lib_pypy/numpypy/lib/ufunclike.py b/lib_pypy/numpypy/lib/ufunclike.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/lib/ufunclike.py
@@ -0,0 +1,177 @@
+"""
+Module of functions that are like ufuncs in acting on arrays and optionally
+storing results in an output array.
+
+"""
+from __future__ import division, absolute_import, print_function
+
+__all__ = ['fix', 'isneginf', 'isposinf']
+
+from ..core import numeric as nx
+
+def fix(x, y=None):
+    """
+    Round to nearest integer towards zero.
+
+    Round an array of floats element-wise to nearest integer towards zero.
+    The rounded values are returned as floats.
+
+    Parameters
+    ----------
+    x : array_like
+        An array of floats to be rounded
+    y : ndarray, optional
+        Output array
+
+    Returns
+    -------
+    out : ndarray of floats
+        The array of rounded numbers
+
+    See Also
+    --------
+    trunc, floor, ceil
+    around : Round to given number of decimals
+
+    Examples
+    --------
+    >>> np.fix(3.14)
+    3.0
+    >>> np.fix(3)
+    3.0
+    >>> np.fix([2.1, 2.9, -2.1, -2.9])
+    array([ 2.,  2., -2., -2.])
+
+    """
+    x = nx.asanyarray(x)
+    y1 = nx.floor(x)
+    y2 = nx.ceil(x)
+    if y is None:
+        y = nx.asanyarray(y1)
+    y[...] = nx.where(x >= 0, y1, y2)
+    return y
+
+def isposinf(x, y=None):
+    """
+    Test element-wise for positive infinity, return result as bool array.
+
+    Parameters
+    ----------
+    x : array_like
+        The input array.
+    y : array_like, optional
+        A boolean array with the same shape as `x` to store the result.
+
+    Returns
+    -------
+    y : ndarray
+        A boolean array with the same dimensions as the input.
+        If second argument is not supplied then a boolean array is returned
+        with values True where the corresponding element of the input is
+        positive infinity and values False where the element of the input is
+        not positive infinity.
+
+        If a second argument is supplied the result is stored there. If the
+        type of that array is a numeric type the result is represented as zeros
+        and ones, if the type is boolean then as False and True.
+        The return value `y` is then a reference to that array.
+
+    See Also
+    --------
+    isinf, isneginf, isfinite, isnan
+
+    Notes
+    -----
+    Numpy uses the IEEE Standard for Binary Floating-Point for Arithmetic
+    (IEEE 754).
+
+    Errors result if the second argument is also supplied when `x` is a
+    scalar input, or if first and second arguments have different shapes.
+
+    Examples
+    --------
+    >>> np.isposinf(np.PINF)
+    array(True, dtype=bool)
+    >>> np.isposinf(np.inf)
+    array(True, dtype=bool)
+    >>> np.isposinf(np.NINF)
+    array(False, dtype=bool)
+    >>> np.isposinf([-np.inf, 0., np.inf])
+    array([False, False,  True], dtype=bool)
+
+    >>> x = np.array([-np.inf, 0., np.inf])
+    >>> y = np.array([2, 2, 2])
+    >>> np.isposinf(x, y)
+    array([0, 0, 1])
+    >>> y
+    array([0, 0, 1])
+
+    """
+    if y is None:
+        x = nx.asarray(x)
+        y = nx.empty(x.shape, dtype=nx.bool_)
+    nx.logical_and(nx.isinf(x), ~nx.signbit(x), y)
+    return y
+
+def isneginf(x, y=None):
+    """
+    Test element-wise for negative infinity, return result as bool array.
+
+    Parameters
+    ----------
+    x : array_like
+        The input array.
+    y : array_like, optional
+        A boolean array with the same shape and type as `x` to store the
+        result.
+
+    Returns
+    -------
+    y : ndarray
+        A boolean array with the same dimensions as the input.
+        If second argument is not supplied then a numpy boolean array is
+        returned with values True where the corresponding element of the
+        input is negative infinity and values False where the element of
+        the input is not negative infinity.
+
+        If a second argument is supplied the result is stored there. If the
+        type of that array is a numeric type the result is represented as
+        zeros and ones, if the type is boolean then as False and True. The
+        return value `y` is then a reference to that array.
+
+    See Also
+    --------
+    isinf, isposinf, isnan, isfinite
+
+    Notes
+    -----
+    Numpy uses the IEEE Standard for Binary Floating-Point for Arithmetic
+    (IEEE 754).
+
+    Errors result if the second argument is also supplied when x is a scalar
+    input, or if first and second arguments have different shapes.
+
+    Examples
+    --------
+    >>> np.isneginf(np.NINF)
+    array(True, dtype=bool)
+    >>> np.isneginf(np.inf)
+    array(False, dtype=bool)
+    >>> np.isneginf(np.PINF)
+    array(False, dtype=bool)
+    >>> np.isneginf([-np.inf, 0., np.inf])
+    array([ True, False, False], dtype=bool)
+
+    >>> x = np.array([-np.inf, 0., np.inf])
+    >>> y = np.array([2, 2, 2])
+    >>> np.isneginf(x, y)
+    array([1, 0, 0])
+    >>> y
+    array([1, 0, 0])
+
+    """
+    if y is None:
+        x = nx.asarray(x)
+        y = nx.empty(x.shape, dtype=nx.bool_)
+    nx.logical_and(nx.isinf(x), nx.signbit(x), y)
+    return y
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -63,6 +63,7 @@
         ("less_equal", "less_equal"),
         ("maximum", "maximum"),
         ("minimum", "minimum"),
+        ("mod", "mod"),
         ("multiply", "multiply"),
         ("negative", "negative"),
         ("not_equal", "not_equal"),
@@ -90,8 +91,6 @@
         ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
-        ('isneginf', 'isneginf'),
-        ('isposinf', 'isposinf'),
         ('isfinite', 'isfinite'),
         ('logical_and', 'logical_and'),
         ('logical_xor', 'logical_xor'),
@@ -105,6 +104,7 @@
         ('floor_divide', 'floor_divide'),
         ('logaddexp', 'logaddexp'),
         ('logaddexp2', 'logaddexp2'),
+        ('ldexp', 'ldexp'),
         ('real', 'real'),
         ('imag', 'imag'),
         ('ones_like', 'ones_like'),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -16,23 +16,24 @@
 def done_if_false(dtype, val):
     return not dtype.itemtype.bool(val)
 
+
 class W_Ufunc(W_Root):
     _attrs_ = ["name", "promote_to_float", "promote_bools", "identity",
-               "allow_complex", "complex_to_float"]
+               "allow_bool", "allow_complex", "complex_to_float"]
     _immutable_fields_ = ["promote_to_float", "promote_bools", "name",
-            "allow_complex", "complex_to_float"]
+            "allow_bool", "allow_complex", "complex_to_float"]
 
     def __init__(self, name, promote_to_float, promote_bools, identity,
-                 int_only, allow_complex, complex_to_float):
+                 int_only, allow_bool, allow_complex, complex_to_float):
         self.name = name
         self.promote_to_float = promote_to_float
         self.promote_bools = promote_bools
+        self.identity = identity
+        self.int_only = int_only
+        self.allow_bool = allow_bool
         self.allow_complex = allow_complex
         self.complex_to_float = complex_to_float
 
-        self.identity = identity
-        self.int_only = int_only
-
     def descr_repr(self, space):
         return space.wrap("<ufunc '%s'>" % self.name)
 
@@ -259,10 +260,10 @@
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
         identity=None, bool_result=False, int_only=False,
-        allow_complex=True, complex_to_float=False):
+        allow_bool=True, allow_complex=True, complex_to_float=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity,
-                         int_only, allow_complex, complex_to_float)
+                         int_only, allow_bool, allow_complex, complex_to_float)
         self.func = func
         self.bool_result = bool_result
 
@@ -274,17 +275,19 @@
             if space.is_w(out, space.w_None):
                 out = None
         w_obj = convert_to_array(space, w_obj)
-        if w_obj.get_dtype().is_flexible_type():
+        dtype = w_obj.get_dtype()
+        if dtype.is_flexible_type():
             raise OperationError(space.w_TypeError,
                       space.wrap('Not implemented for this type'))
-        if self.int_only and not w_obj.get_dtype().is_int_type():
+        if (self.int_only and not dtype.is_int_type() or
+                not self.allow_bool and dtype.is_bool_type() or
+                not self.allow_complex and dtype.is_complex_type()):
             raise OperationError(space.w_TypeError, space.wrap(
                 "ufunc %s not supported for the input type" % self.name))
         calc_dtype = find_unaryop_result_dtype(space,
                                   w_obj.get_dtype(),
                                   promote_to_float=self.promote_to_float,
-                                  promote_bools=self.promote_bools,
-                                  allow_complex=self.allow_complex)
+                                  promote_bools=self.promote_bools)
         if out is not None:
             if not isinstance(out, W_NDimArray):
                 raise OperationError(space.w_TypeError, space.wrap(
@@ -324,10 +327,10 @@
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
         identity=None, comparison_func=False, int_only=False,
-        allow_complex=True, complex_to_float=False):
+        allow_bool=True, allow_complex=True, complex_to_float=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity,
-                         int_only, allow_complex, complex_to_float)
+                         int_only, allow_bool, allow_complex, complex_to_float)
         self.func = func
         self.comparison_func = comparison_func
         if name == 'logical_and':
@@ -375,16 +378,14 @@
                 w_rdtype = w_ldtype
             elif w_lhs.is_scalar() and not w_rhs.is_scalar():
                 w_ldtype = w_rdtype
+        if (self.int_only and (not w_ldtype.is_int_type() or not w_rdtype.is_int_type()) or
+                not self.allow_bool and (w_ldtype.is_bool_type() or w_rdtype.is_bool_type()) or
+                not self.allow_complex and (w_ldtype.is_complex_type() or w_rdtype.is_complex_type())):
+            raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
         calc_dtype = find_binop_result_dtype(space,
             w_ldtype, w_rdtype,
-            int_only=self.int_only,
             promote_to_float=self.promote_to_float,
-            promote_bools=self.promote_bools,
-            allow_complex=self.allow_complex,
-            )
-        if self.int_only and not calc_dtype.is_int_type():
-            raise OperationError(space.w_TypeError, space.wrap(
-                "ufunc '%s' not supported for the input types" % self.name))
+            promote_bools=self.promote_bools)
         if space.is_none(w_out):
             out = None
         elif not isinstance(w_out, W_NDimArray):
@@ -431,14 +432,10 @@
 
 
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
-    promote_bools=False, int_only=False, allow_complex=True):
+        promote_bools=False):
     # dt1.num should be <= dt2.num
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
-    if int_only and (not dt1.is_int_type() or not dt2.is_int_type()):
-        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
-    if not allow_complex and (dt1.is_complex_type() or dt2.is_complex_type()):
-        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
     # Some operations promote op(bool, bool) to return int8, rather than bool
     if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
@@ -507,14 +504,11 @@
         dtypenum += 2
         return interp_dtype.get_dtype_cache(space).dtypes_by_num[dtypenum]
 
-
 @jit.unroll_safe
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
-    promote_bools=False, promote_to_largest=False, allow_complex=True):
+        promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
-    if not allow_complex and (dt.is_complex_type()):
-        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
     if promote_to_float:
         if dt.kind == interp_dtype.FLOATINGLTR or dt.kind==interp_dtype.COMPLEXLTR:
             return dt
@@ -535,7 +529,6 @@
             assert False
     return dt
 
-
 def find_dtype_for_scalar(space, w_obj, current_guess=None):
     bool_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
     long_dtype = interp_dtype.get_dtype_cache(space).w_longdtype
@@ -588,7 +581,6 @@
         'unable to create dtype from objects, ' '"%T" instance not supported',
         w_obj)
 
-
 def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func,
                        bool_result):
     dtype_cache = interp_dtype.get_dtype_cache(space)
@@ -606,6 +598,7 @@
             return res
     return func_with_new_name(impl, ufunc_name)
 
+
 class UfuncState(object):
     def __init__(self, space):
         "NOT_RPYTHON"
@@ -635,10 +628,6 @@
             ("greater_equal", "ge", 2, {"comparison_func": True}),
             ("isnan", "isnan", 1, {"bool_result": True}),
             ("isinf", "isinf", 1, {"bool_result": True}),
-            ("isneginf", "isneginf", 1, {"bool_result": True,
-                                         "allow_complex": False}),
-            ("isposinf", "isposinf", 1, {"bool_result": True,
-                                         "allow_complex": False}),
             ("isfinite", "isfinite", 1, {"bool_result": True}),
 
             ('logical_and', 'logical_and', 2, {'comparison_func': True,
@@ -658,7 +647,7 @@
             ("negative", "neg", 1),
             ("absolute", "abs", 1, {"complex_to_float": True}),
             ("rint", "rint", 1),
-            ("sign", "sign", 1, {"promote_bools": True}),
+            ("sign", "sign", 1, {"allow_bool": False}),
             ("signbit", "signbit", 1, {"bool_result": True,
                                        "allow_complex": False}),
             ("reciprocal", "reciprocal", 1),
@@ -713,6 +702,8 @@
                                        "allow_complex": False}),
             ("logaddexp2", "logaddexp2", 2, {"promote_to_float": True,
                                        "allow_complex": False}),
+            ("ldexp", "ldexp", 2, {"int_only": True}),
+
             ("ones_like", "ones_like", 1),
             ("zeros_like", "zeros_like", 1),
         ]:
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -91,14 +91,14 @@
                         uncallable.add(s)
             return uncallable
         assert find_uncallable_ufuncs('int') == set()
-        assert find_uncallable_ufuncs('bool') == set()
+        assert find_uncallable_ufuncs('bool') == set(['sign'])
         assert find_uncallable_ufuncs('float') == set(
                 ['bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor',
-                 'left_shift', 'right_shift', 'invert'])
+                 'left_shift', 'right_shift', 'invert', 'ldexp'])
         assert find_uncallable_ufuncs('complex') == set(
                 ['bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor',
                  'arctan2', 'deg2rad', 'degrees', 'rad2deg', 'radians',
-                 'fabs', 'fmod', 'invert', 'isneginf', 'isposinf',
+                 'fabs', 'fmod', 'invert', 'ldexp', 'mod',
                  'logaddexp', 'logaddexp2', 'left_shift', 'right_shift',
                  'copysign', 'signbit', 'ceil', 'floor', 'trunc'])
 
@@ -174,7 +174,6 @@
         assert fabs(float('-inf')) == float('inf')
         assert isnan(fabs(float('nan')))
 
-
     def test_fmax(self):
         from numpypy import fmax, array
         import math
@@ -194,7 +193,6 @@
         # on Microsoft win32
         assert math.copysign(1., fmax(nnan, nan)) == math.copysign(1., nnan)
 
-
     def test_fmin(self):
         from numpypy import fmin, array
         import math
@@ -213,7 +211,6 @@
         # on Microsoft win32
         assert math.copysign(1., fmin(nnan, nan)) == math.copysign(1., nnan)
 
-
     def test_fmod(self):
         from numpypy import fmod
         import math
@@ -368,7 +365,6 @@
         c = array([10.5+11.5j, -15.2-100.3456j, 0.2343+11.123456j])
         assert (c.round(0) == [10.+12.j, -15-100j, 0+11j]).all()
 
-
     def test_copysign(self):
         from numpypy import array, copysign
 
@@ -436,7 +432,6 @@
 
         assert expm1(1e-50) == 1e-50
 
-
     def test_sin(self):
         import math
         from numpypy import array, sin
@@ -704,6 +699,8 @@
         assert (~a == [-2, -3, -4, -5]).all()
         assert (bitwise_not(a) == ~a).all()
         assert (invert(a) == ~a).all()
+        assert invert(True) == False
+        assert invert(False) == True
 
     def test_shift(self):
         from numpypy import left_shift, right_shift, bool
@@ -964,6 +961,11 @@
         assert logaddexp2(float('inf'), float('-inf')) == float('inf')
         assert logaddexp2(float('inf'), float('inf')) == float('inf')
 
+    def test_ldexp(self):
+        import numpypy as np
+        a = np.ldexp(2, 3)
+        assert type(a) is np.float64 and a == 16.0
+
     def test_ones_like(self):
         from numpypy import array, ones_like
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -248,14 +248,6 @@
     def isinf(self, v):
         return False
 
-    @raw_unary_op
-    def isneginf(self, v):
-        return False
-
-    @raw_unary_op
-    def isposinf(self, v):
-        return False
-
     @raw_binary_op
     def eq(self, v1, v2):
         return v1 == v2
@@ -320,6 +312,10 @@
         float64 = Float64()
         return float64.rint(float64.box(v))
 
+    @raw_binary_op
+    def ldexp(self, v1, v2):
+        return Float64().box(v1 * 2**v2)
+
 class NonNativePrimitive(Primitive):
     _mixin_ = True
 
@@ -401,7 +397,7 @@
 
     @simple_unary_op
     def invert(self, v):
-        return ~v
+        return not v
 
     @raw_unary_op
     def isfinite(self, v):
@@ -497,14 +493,6 @@
     def isinf(self, v):
         return False
 
-    @raw_unary_op
-    def isposinf(self, v):
-        return False
-
-    @raw_unary_op
-    def isneginf(self, v):
-        return False
-
     @simple_binary_op
     def bitwise_and(self, v1, v2):
         return v1 & v2
@@ -947,14 +935,6 @@
         return rfloat.isinf(v)
 
     @raw_unary_op
-    def isneginf(self, v):
-        return rfloat.isinf(v) and v < 0
-
-    @raw_unary_op
-    def isposinf(self, v):
-        return rfloat.isinf(v) and v > 0
-
-    @raw_unary_op
     def isfinite(self, v):
         return not (rfloat.isinf(v) or rfloat.isnan(v))
 

From noreply at buildbot.pypy.org  Thu Oct 17 09:36:30 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 09:36:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: oops ldexp breaks translation,
	remove for now
Message-ID: <20131017073630.A73381D232F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67455:c3f5e0155683
Date: 2013-10-17 03:31 -0400
http://bitbucket.org/pypy/pypy/changeset/c3f5e0155683/

Log:	oops ldexp breaks translation, remove for now

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -104,7 +104,6 @@
         ('floor_divide', 'floor_divide'),
         ('logaddexp', 'logaddexp'),
         ('logaddexp2', 'logaddexp2'),
-        ('ldexp', 'ldexp'),
         ('real', 'real'),
         ('imag', 'imag'),
         ('ones_like', 'ones_like'),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -702,7 +702,6 @@
                                        "allow_complex": False}),
             ("logaddexp2", "logaddexp2", 2, {"promote_to_float": True,
                                        "allow_complex": False}),
-            ("ldexp", "ldexp", 2, {"int_only": True}),
 
             ("ones_like", "ones_like", 1),
             ("zeros_like", "zeros_like", 1),
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -94,11 +94,11 @@
         assert find_uncallable_ufuncs('bool') == set(['sign'])
         assert find_uncallable_ufuncs('float') == set(
                 ['bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor',
-                 'left_shift', 'right_shift', 'invert', 'ldexp'])
+                 'left_shift', 'right_shift', 'invert'])
         assert find_uncallable_ufuncs('complex') == set(
                 ['bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor',
                  'arctan2', 'deg2rad', 'degrees', 'rad2deg', 'radians',
-                 'fabs', 'fmod', 'invert', 'ldexp', 'mod',
+                 'fabs', 'fmod', 'invert', 'mod',
                  'logaddexp', 'logaddexp2', 'left_shift', 'right_shift',
                  'copysign', 'signbit', 'ceil', 'floor', 'trunc'])
 
@@ -961,11 +961,6 @@
         assert logaddexp2(float('inf'), float('-inf')) == float('inf')
         assert logaddexp2(float('inf'), float('inf')) == float('inf')
 
-    def test_ldexp(self):
-        import numpypy as np
-        a = np.ldexp(2, 3)
-        assert type(a) is np.float64 and a == 16.0
-
     def test_ones_like(self):
         from numpypy import array, ones_like
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -312,10 +312,6 @@
         float64 = Float64()
         return float64.rint(float64.box(v))
 
-    @raw_binary_op
-    def ldexp(self, v1, v2):
-        return Float64().box(v1 * 2**v2)
-
 class NonNativePrimitive(Primitive):
     _mixin_ = True
 

From noreply at buildbot.pypy.org  Thu Oct 17 09:36:31 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 09:36:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: don't lie about size of long double,
 computations are currently done with double regardless
Message-ID: <20131017073631.D23CD1D232F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67456:f01d9caa632c
Date: 2013-10-17 03:33 -0400
http://bitbucket.org/pypy/pypy/changeset/f01d9caa632c/

Log:	don't lie about size of long double, computations are currently done
	with double regardless

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -20,11 +20,14 @@
 MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
 MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else ()
 
-long_double_size = rffi.sizeof_c_type('long double', ignore_errors=True)
-import os
-if long_double_size == 8 and os.name == 'nt':
-    # this is a lie, or maybe a wish, MS fakes longdouble math with double
-    long_double_size = 12
+#long_double_size = rffi.sizeof_c_type('long double', ignore_errors=True)
+#import os
+#if long_double_size == 8 and os.name == 'nt':
+#    # this is a lie, or maybe a wish, MS fakes longdouble math with double
+#    long_double_size = 12
+
+# hardcode to 8 for now (simulate using normal double) until long double works
+long_double_size = 8
 
 
 def new_dtype_getter(name):

From noreply at buildbot.pypy.org  Thu Oct 17 10:06:06 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 10:06:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix __reduce__() order for record arrays
Message-ID: <20131017080606.879451C1161@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67457:09b36515b085
Date: 2013-10-17 04:04 -0400
http://bitbucket.org/pypy/pypy/changeset/09b36515b085/

Log:	fix __reduce__() order for record arrays

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -259,21 +259,22 @@
         builder_args = space.newtuple([space.wrap("%s%d" % (kind, elemsize)), space.wrap(0), space.wrap(1)])
 
         version = space.wrap(3)
-        order = space.wrap(byteorder_prefix if self.native else nonnative_byteorder_prefix)
         names = self.descr_get_names(space)
         values = self.descr_get_fields(space)
         if self.fields:
+            order = space.wrap('|')
             #TODO: Implement this when subarrays are implemented
             subdescr = space.w_None
-            #TODO: Change this when alignment is implemented :
             size = 0
             for key in self.fields:
                 dtype = self.fields[key][1]
                 assert isinstance(dtype, W_Dtype)
                 size += dtype.get_size()
             w_size = space.wrap(size)
+            #TODO: Change this when alignment is implemented
             alignment = space.wrap(1)
         else:
+            order = space.wrap(byteorder_prefix if self.native else nonnative_byteorder_prefix)
             subdescr = space.w_None
             w_size = space.wrap(-1)
             alignment = space.wrap(-1)
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -885,7 +885,7 @@
         from cPickle import loads, dumps
 
         d = dtype([("x", "int32"), ("y", "int32"), ("z", "int32"), ("value", float)])
-        assert d.__reduce__() == (dtype, ('V20', 0, 1), (3, '<', None, ('x', 'y', 'z', 'value'), {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0), 'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12)}, 20, 1, 0))
+        assert d.__reduce__() == (dtype, ('V20', 0, 1), (3, '|', None, ('x', 'y', 'z', 'value'), {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0), 'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12)}, 20, 1, 0))
 
         new_d = loads(dumps(d))
 

From noreply at buildbot.pypy.org  Thu Oct 17 16:34:18 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 17 Oct 2013 16:34:18 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: some renames
Message-ID: <20131017143418.BDAD61C30DC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67458:f68ab0b1e774
Date: 2013-10-17 16:33 +0200
http://bitbucket.org/pypy/pypy/changeset/f68ab0b1e774/

Log:	some renames

diff --git a/rpython/jit/backend/llsupport/test/test_resume.py b/rpython/jit/backend/llsupport/test/test_resumebuilder.py
rename from rpython/jit/backend/llsupport/test/test_resume.py
rename to rpython/jit/backend/llsupport/test/test_resumebuilder.py
diff --git a/rpython/jit/backend/x86/test/test_resume.py b/rpython/jit/backend/x86/test/test_resumebuilder.py
rename from rpython/jit/backend/x86/test/test_resume.py
rename to rpython/jit/backend/x86/test/test_resumebuilder.py

From noreply at buildbot.pypy.org  Thu Oct 17 16:36:14 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 17 Oct 2013 16:36:14 +0200 (CEST)
Subject: [pypy-commit] pypy resume-refactor: fix
Message-ID: <20131017143614.5295B1C30DC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: resume-refactor
Changeset: r67459:1fd12147c3fc
Date: 2013-10-17 16:35 +0200
http://bitbucket.org/pypy/pypy/changeset/1fd12147c3fc/

Log:	fix

diff --git a/rpython/jit/backend/x86/test/test_resumebuilder.py b/rpython/jit/backend/x86/test/test_resumebuilder.py
--- a/rpython/jit/backend/x86/test/test_resumebuilder.py
+++ b/rpython/jit/backend/x86/test/test_resumebuilder.py
@@ -1,6 +1,6 @@
 
 from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
-from rpython.jit.backend.llsupport.test.test_resume import ResumeTest
+from rpython.jit.backend.llsupport.test.test_resumebuilder import ResumeTest
 
 class TestResumeX86(Jit386Mixin, ResumeTest):
     # for the individual tests see

From noreply at buildbot.pypy.org  Thu Oct 17 16:43:46 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 17 Oct 2013 16:43:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix test_gdb_pypy (sorry)
Message-ID: <20131017144346.383551C1161@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67460:ddfba0b56c0e
Date: 2013-10-17 16:43 +0200
http://bitbucket.org/pypy/pypy/changeset/ddfba0b56c0e/

Log:	Fix test_gdb_pypy (sorry)

diff --git a/pypy/tool/test/test_gdb_pypy.py b/pypy/tool/test/test_gdb_pypy.py
--- a/pypy/tool/test/test_gdb_pypy.py
+++ b/pypy/tool/test/test_gdb_pypy.py
@@ -1,4 +1,4 @@
-import py, sys
+import py, sys, zlib, re
 from pypy.tool import gdb_pypy
 
 class FakeGdb(object):
@@ -9,17 +9,30 @@
     TYPE_CODE_ARRAY = 2
     TYPE_CODE_STRUCT = 3
 
-    def __init__(self, exprs, progspace=None):
+    def __init__(self, typeids, exprs):
+        self.typeids_z = zlib.compress(typeids)
+        exprs['*(long*)pypy_g_rpython_memory_gctypelayout_GCData'
+              '.gcd_inst_typeids_z'] = len(self.typeids_z)
         self.exprs = exprs
-        self.progspace = progspace
         self._parsed = []
 
     def parse_and_eval(self, expr):
         self._parsed.append(expr)
         return self.exprs[expr]
 
-    def current_progspace(self):
-        return self.progspace
+    def execute(self, command):
+        r = re.compile(r"dump binary memory (\S+) (\S+) (\S+)$")
+        match = r.match(command)
+        assert match
+        fn, start, stop = match.groups()
+        assert start == (
+            '(char*)(((long*)pypy_g_rpython_memory_gctypelayout_GCData'
+            '.gcd_inst_typeids_z)+1)')
+        assert stop == (
+            '(char*)(((long*)pypy_g_rpython_memory_gctypelayout_GCData'
+            '.gcd_inst_typeids_z)+1)+%d' % (len(self.typeids_z),))
+        with open(fn, 'wb') as f:
+            f.write(self.typeids_z)
 
 
 class Mock(object):
@@ -111,33 +124,27 @@
             % (TIDT, n, TIDT))
 
 def test_load_typeids(tmpdir):
-    exe = tmpdir.join('testing_1').join('pypy-c')
-    typeids = tmpdir.join('typeids.txt')
-    typeids.write("""
+    typeids = """
 member0    ?
 member1    GcStruct xxx {}
-""".strip())
-    progspace = Mock(filename=str(exe))
+""".lstrip()
     exprs = {exprmember(1): 111}
-    gdb = FakeGdb(exprs, progspace)
+    gdb = FakeGdb(typeids, exprs)
     cmd = gdb_pypy.RPyType(gdb)
-    typeids = cmd.load_typeids(progspace)
+    typeids = cmd.load_typeids()
     assert typeids[0] == '(null typeid)'
     assert typeids[111] == 'GcStruct xxx {}'
     py.test.raises(KeyError, "typeids[50]")
     py.test.raises(KeyError, "typeids[150]")
 
 def test_RPyType(tmpdir):
-    exe = tmpdir.join('pypy-c')
-    typeids = tmpdir.join('typeids.txt')
-    typeids.write("""
+    typeids = """
 member0    ?
 member1    GcStruct xxx {}
 member2    GcStruct yyy {}
 member3    GcStruct zzz {}
-""".strip())
+""".lstrip()
     #
-    progspace = Mock(filename=str(exe))
     d = {'r_super': {
             '_gcheader': {
                 'h_tid': 123,
@@ -152,7 +159,7 @@
         exprmember(2): 123,
         exprmember(3): 456,
         }
-    gdb = FakeGdb(exprs, progspace)
+    gdb = FakeGdb(typeids, exprs)
     cmd = gdb_pypy.RPyType(gdb)
     assert cmd.do_invoke('*myvar', True) == 'GcStruct yyy {}'
 
@@ -198,9 +205,9 @@
     assert gdb_pypy.RPyListPrinter.lookup(mylist, FakeGdb) is None
 
 def test_typeidsmap():
-    gdb = FakeGdb({exprmember(1): 111,
-                   exprmember(2): 222,
-                   exprmember(3): 333})
+    gdb = FakeGdb('', {exprmember(1): 111,
+                       exprmember(2): 222,
+                       exprmember(3): 333})
     typeids = gdb_pypy.TypeIdsMap(["member0  ?\n",
                                    "member1  FooBar\n",
                                    "member2  Baz\n",

From noreply at buildbot.pypy.org  Thu Oct 17 16:45:37 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 17 Oct 2013 16:45:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: Maybe fix the test for 32-bit
Message-ID: <20131017144537.43A4A1C1161@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67461:0138c06ab49c
Date: 2013-10-17 16:44 +0200
http://bitbucket.org/pypy/pypy/changeset/0138c06ab49c/

Log:	Maybe fix the test for 32-bit

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -245,7 +245,7 @@
         guard_not_invalidated(descr=...)
         p52 = call(ConstClass(str_decode_ascii__raise_unicode_exception_decode), ConstPtr(ptr38), 3, 1, descr=<Callr . rii EF=4>)
         guard_no_exception(descr=...)
-        p53 = getfield_gc_pure(p52, descr=<FieldP tuple2.item0 8>)
+        p53 = getfield_gc_pure(p52, descr=<FieldP tuple2.item0 .>)
         guard_nonnull(p53, descr=...)
         --TICK--
         jump(..., descr=...)

From noreply at buildbot.pypy.org  Thu Oct 17 17:08:54 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 17 Oct 2013 17:08:54 +0200 (CEST)
Subject: [pypy-commit] cffi default: Support partial unions in a way very
 similar to partial structs.
Message-ID: <20131017150854.A87941C30DC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1375:857a065e4e0c
Date: 2013-10-17 17:07 +0200
http://bitbucket.org/cffi/cffi/changeset/857a065e4e0c/

Log:	Support partial unions in a way very similar to partial structs.
	Needed for a python-cffi mail.

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -488,7 +488,7 @@
         return tp
 
     def _make_partial(self, tp, nested):
-        if not isinstance(tp, model.StructType):
+        if not isinstance(tp, model.StructOrUnion):
             raise api.CDefError("%s cannot be partial" % (tp,))
         if not tp.has_c_name() and not nested:
             raise NotImplementedError("%s is partial but has no C name" %(tp,))
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -248,6 +248,7 @@
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
     completed = False
+    partial = False
 
     def __init__(self, name, fldnames, fldtypes, fldbitsize):
         self.name = name
@@ -344,11 +345,6 @@
         from .ffiplatform import VerificationError
         raise VerificationError(msg)
 
-
-class StructType(StructOrUnion):
-    kind = 'struct'
-    partial = False
-
     def check_not_partial(self):
         if self.partial and self.fixedlayout is None:
             from . import ffiplatform
@@ -357,19 +353,18 @@
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        
-        return global_cache(self, ffi, 'new_struct_type',
+        #
+        return global_cache(self, ffi, 'new_%s_type' % self.kind,
                             self.get_official_name(), key=self)
 
 
+class StructType(StructOrUnion):
+    kind = 'struct'
+
+
 class UnionType(StructOrUnion):
     kind = 'union'
 
-    def build_backend_type(self, ffi, finishlist):
-        finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type',
-                            self.get_official_name(), key=self)
-
 
 class EnumType(StructOrUnionOrEnum):
     kind = 'enum'
diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -491,7 +491,7 @@
         #
         function = getattr(module, layoutfuncname)
         layout = function()
-        if isinstance(tp, model.StructType) and tp.partial:
+        if isinstance(tp, model.StructOrUnion) and tp.partial:
             # use the function()'s sizes and offsets to guide the
             # layout of the struct
             totalsize = layout[0]
diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -282,7 +282,7 @@
             if x < 0: break
             layout.append(x)
             num += 1
-        if isinstance(tp, model.StructType) and tp.partial:
+        if isinstance(tp, model.StructOrUnion) and tp.partial:
             # use the function()'s sizes and offsets to guide the
             # layout of the struct
             totalsize = layout[0]
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -1162,6 +1162,36 @@
     ffi.cdef("union foo_u { char x; long *z; };")
     ffi.verify("union foo_u { char x; int y; long *z; };")
 
+def test_ffi_union_partial():
+    ffi = FFI()
+    ffi.cdef("union foo_u { char x; ...; };")
+    ffi.verify("union foo_u { char x; int y; };")
+    assert ffi.sizeof("union foo_u") == 4
+
+def test_ffi_union_with_partial_struct():
+    ffi = FFI()
+    ffi.cdef("struct foo_s { int x; ...; }; union foo_u { struct foo_s s; };")
+    ffi.verify("struct foo_s { int a; int x; }; "
+               "union foo_u { char b[32]; struct foo_s s; };")
+    assert ffi.sizeof("struct foo_s") == 8
+    assert ffi.sizeof("union foo_u") == 32
+
+def test_ffi_union_partial_2():
+    ffi = FFI()
+    ffi.cdef("typedef union { char x; ...; } u1;")
+    ffi.verify("typedef union { char x; int y; } u1;")
+    assert ffi.sizeof("u1") == 4
+
+def test_ffi_union_with_partial_struct_2():
+    ffi = FFI()
+    ffi.cdef("typedef struct { int x; ...; } s1;"
+             "typedef union { s1 s; } u1;")
+    ffi.verify("typedef struct { int a; int x; } s1; "
+               "typedef union { char b[32]; s1 s; } u1;")
+    assert ffi.sizeof("s1") == 8
+    assert ffi.sizeof("u1") == 32
+    assert ffi.offsetof("u1", "s") == 0
+
 def test_ffi_struct_packed():
     if sys.platform == 'win32':
         py.test.skip("needs a GCC extension")

From noreply at buildbot.pypy.org  Thu Oct 17 17:40:40 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 17 Oct 2013 17:40:40 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Merge default
Message-ID: <20131017154040.6321B1C30DC@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-array_prepare_-array_wrap
Changeset: r67462:18c46a0fd930
Date: 2013-10-17 17:39 +0200
http://bitbucket.org/pypy/pypy/changeset/18c46a0fd930/

Log:	Merge default

diff too long, truncating to 2000 out of 3739 lines

diff --git a/TODO.txt b/TODO.txt
deleted file mode 100644
--- a/TODO.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-TODO list by mattip
-===================
-
-- test, implement use of __array_prepare__()
-- test, implement use of __array_wrap__()
diff --git a/lib-python/2.7/test/keycert.pem b/lib-python/2.7/test/keycert.pem
--- a/lib-python/2.7/test/keycert.pem
+++ b/lib-python/2.7/test/keycert.pem
@@ -1,32 +1,31 @@
------BEGIN RSA PRIVATE KEY-----
-MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L
-opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH
-fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB
-AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU
-D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA
-IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM
-oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0
-ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/
-loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j
-oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA
-z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq
-ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV
-q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU=
------END RSA PRIVATE KEY-----
+-----BEGIN PRIVATE KEY-----
+MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm
+LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0
+ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP
+USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt
+CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq
+SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK
+UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y
+BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ
+ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5
+oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik
+eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F
+0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS
+x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/
+SPIXQuT8RMPDVNQ=
+-----END PRIVATE KEY-----
 -----BEGIN CERTIFICATE-----
-MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD
-VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x
-IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT
-U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1
-NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl
-bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m
-dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj
-aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh
-m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8
-M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn
-fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC
-AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb
-08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx
-CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/
-iHkC6gGdBJhogs4=
+MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw
+MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH
+Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k
+YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
+gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7
+6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt
+pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw
+FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd
+BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G
+lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1
+CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX
 -----END CERTIFICATE-----
diff --git a/lib-python/2.7/test/sha256.pem b/lib-python/2.7/test/sha256.pem
--- a/lib-python/2.7/test/sha256.pem
+++ b/lib-python/2.7/test/sha256.pem
@@ -1,129 +1,128 @@
 # Certificate chain for https://sha256.tbs-internet.com
- 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=sha-256 production/CN=sha256.tbs-internet.com
-   i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC
+ 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=Certificats TBS X509/CN=ecom.tbs-x509.com
+   i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business
 -----BEGIN CERTIFICATE-----
-MIIGXTCCBUWgAwIBAgIRAMmag+ygSAdxZsbyzYjhuW0wDQYJKoZIhvcNAQELBQAw
-gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl
+MIIGTjCCBTagAwIBAgIQOh3d9dNDPq1cSdJmEiMpqDANBgkqhkiG9w0BAQUFADCB
+yTELMAkGA1UEBhMCRlIxETAPBgNVBAgTCENhbHZhZG9zMQ0wCwYDVQQHEwRDYWVu
+MRUwEwYDVQQKEwxUQlMgSU5URVJORVQxSDBGBgNVBAsTP1Rlcm1zIGFuZCBDb25k
+aXRpb25zOiBodHRwOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0EvcmVwb3NpdG9y
+eTEYMBYGA1UECxMPVEJTIElOVEVSTkVUIENBMR0wGwYDVQQDExRUQlMgWDUwOSBD
+QSBidXNpbmVzczAeFw0xMTAxMjUwMDAwMDBaFw0xMzAyMDUyMzU5NTlaMIHHMQsw
+CQYDVQQGEwJGUjEOMAwGA1UEERMFMTQwMDAxETAPBgNVBAgTCENhbHZhZG9zMQ0w
+CwYDVQQHEwRDQUVOMRswGQYDVQQJExIyMiBydWUgZGUgQnJldGFnbmUxFTATBgNV
+BAoTDFRCUyBJTlRFUk5FVDEXMBUGA1UECxMOMDAwMiA0NDA0NDM4MTAxHTAbBgNV
+BAsTFENlcnRpZmljYXRzIFRCUyBYNTA5MRowGAYDVQQDExFlY29tLnRicy14NTA5
+LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKRrlHUnJ++1lpcg
+jtYco7cdmRe+EEfTmwPfCdfV3G1QfsTSvY6FfMpm/83pqHfT+4ANwr18wD9ZrAEN
+G16mf9VdCGK12+TP7DmqeZyGIqlFFoahQnmb8EarvE43/1UeQ2CV9XmzwZvpqeli
+LfXsFonawrY3H6ZnMwS64St61Z+9gdyuZ/RbsoZBbT5KUjDEG844QRU4OT1IGeEI
+eY5NM5RNIh6ZNhVtqeeCxMS7afONkHQrOco73RdSTRck/Hj96Ofl3MHNHryr+AMK
+DGFk1kLCZGpPdXtkxXvaDeQoiYDlil26CWc+YK6xyDPMdsWvoG14ZLyCpzMXA7/7
+4YAQRH0CAwEAAaOCAjAwggIsMB8GA1UdIwQYMBaAFBoJBMz5CY+7HqDO1KQUf0vV
+I1jNMB0GA1UdDgQWBBQgOU8HsWzbmD4WZP5Wtdw7jca2WDAOBgNVHQ8BAf8EBAMC
+BaAwDAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw
+TAYDVR0gBEUwQzBBBgsrBgEEAYDlNwIBATAyMDAGCCsGAQUFBwIBFiRodHRwczov
+L3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL0NQUzEwdwYDVR0fBHAwbjA3oDWgM4Yx
+aHR0cDovL2NybC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNy
+bDAzoDGgL4YtaHR0cDovL2NybC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5l
+c3MuY3JsMIGwBggrBgEFBQcBAQSBozCBoDA9BggrBgEFBQcwAoYxaHR0cDovL2Ny
+dC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNydDA5BggrBgEF
+BQcwAoYtaHR0cDovL2NydC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5lc3Mu
+Y3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wMwYDVR0R
+BCwwKoIRZWNvbS50YnMteDUwOS5jb22CFXd3dy5lY29tLnRicy14NTA5LmNvbTAN
+BgkqhkiG9w0BAQUFAAOCAQEArT4NHfbY87bGAw8lPV4DmHlmuDuVp/y7ltO3Ynse
+3Rz8RxW2AzuO0Oy2F0Cu4yWKtMyEyMXyHqWtae7ElRbdTu5w5GwVBLJHClCzC8S9
+SpgMMQTx3Rgn8vjkHuU9VZQlulZyiPK7yunjc7c310S9FRZ7XxOwf8Nnx4WnB+No
+WrfApzhhQl31w+RyrNxZe58hCfDDHmevRvwLjQ785ZoQXJDj2j3qAD4aI2yB8lB5
+oaE1jlCJzC7Kmz/Y9jzfmv/zAs1LQTm9ktevv4BTUFaGjv9jxnQ1xnS862ZiouLW
+zZYIlYPf4F6JjXGiIQgQRglILUfq3ftJd9/ok9W9ZF8h8w==
+-----END CERTIFICATE-----
+ 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business
+   i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root
+-----BEGIN CERTIFICATE-----
+MIIFPzCCBCegAwIBAgIQDlBz/++iRSmLDeVRHT/hADANBgkqhkiG9w0BAQUFADBv
+MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk
+ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF
+eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDcwOTE4MTkyMlow
+gckxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl
 bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u
 ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv
-cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg
-Q0EgU0dDMB4XDTEwMDIxODAwMDAwMFoXDTEyMDIxOTIzNTk1OVowgcsxCzAJBgNV
-BAYTAkZSMQ4wDAYDVQQREwUxNDAwMDERMA8GA1UECBMIQ2FsdmFkb3MxDTALBgNV
-BAcTBENBRU4xGzAZBgNVBAkTEjIyIHJ1ZSBkZSBCcmV0YWduZTEVMBMGA1UEChMM
-VEJTIElOVEVSTkVUMRcwFQYDVQQLEw4wMDAyIDQ0MDQ0MzgxMDEbMBkGA1UECxMS
-c2hhLTI1NiBwcm9kdWN0aW9uMSAwHgYDVQQDExdzaGEyNTYudGJzLWludGVybmV0
-LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKbuM8VT7f0nntwu
-N3F7v9KIBlhKNAxqCrziOXU5iqUt8HrQB3DtHbdmII+CpVUlwlmepsx6G+srEZ9a
-MIGAy0nxi5aLb7watkyIdPjJTMvTUBQ/+RPWzt5JtYbbY9BlJ+yci0dctP74f4NU
-ISLtlrEjUbf2gTohLrcE01TfmOF6PDEbB5PKDi38cB3NzKfizWfrOaJW6Q1C1qOJ
-y4/4jkUREX1UFUIxzx7v62VfjXSGlcjGpBX1fvtABQOSLeE0a6gciDZs1REqroFf
-5eXtqYphpTa14Z83ITXMfgg5Nze1VtMnzI9Qx4blYBw4dgQVEuIsYr7FDBOITDzc
-VEVXZx0CAwEAAaOCAj8wggI7MB8GA1UdIwQYMBaAFAdEdoWTKLx/bXjSCuv6TEvf
-2YIfMB0GA1UdDgQWBBSJKI/AYVI9RQNY0QPIqc8ej2QivTAOBgNVHQ8BAf8EBAMC
-BaAwDAYDVR0TAQH/BAIwADA0BgNVHSUELTArBggrBgEFBQcDAQYIKwYBBQUHAwIG
-CisGAQQBgjcKAwMGCWCGSAGG+EIEATBMBgNVHSAERTBDMEEGCysGAQQBgOU3AgQB
-MDIwMAYIKwYBBQUHAgEWJGh0dHBzOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0Ev
-Q1BTNDBtBgNVHR8EZjBkMDKgMKAuhixodHRwOi8vY3JsLnRicy1pbnRlcm5ldC5j
-b20vVEJTWDUwOUNBU0dDLmNybDAuoCygKoYoaHR0cDovL2NybC50YnMteDUwOS5j
-b20vVEJTWDUwOUNBU0dDLmNybDCBpgYIKwYBBQUHAQEEgZkwgZYwOAYIKwYBBQUH
-MAKGLGh0dHA6Ly9jcnQudGJzLWludGVybmV0LmNvbS9UQlNYNTA5Q0FTR0MuY3J0
-MDQGCCsGAQUFBzAChihodHRwOi8vY3J0LnRicy14NTA5LmNvbS9UQlNYNTA5Q0FT
-R0MuY3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wPwYD
-VR0RBDgwNoIXc2hhMjU2LnRicy1pbnRlcm5ldC5jb22CG3d3dy5zaGEyNTYudGJz
-LWludGVybmV0LmNvbTANBgkqhkiG9w0BAQsFAAOCAQEAA5NL0D4QSqhErhlkdPmz
-XtiMvdGL+ZehM4coTRIpasM/Agt36Rc0NzCvnQwKE+wkngg1Gy2qe7Q0E/ziqBtB
-fZYzdVgu1zdiL4kTaf+wFKYAFGsFbyeEmXysy+CMwaNoF2vpSjCU1UD56bEnTX/W
-fxVZYxtBQUpnu2wOsm8cDZuZRv9XrYgAhGj9Tt6F0aVHSDGn59uwShG1+BVF/uju
-SCyPTTjL1oc7YElJUzR/x4mQJYvtQI8gDIDAGEOs7v3R/gKa5EMfbUQUI4C84UbI
-Yz09Jdnws/MkC/Hm1BZEqk89u7Hvfv+oHqEb0XaUo0TDfsxE0M1sMdnLb91QNQBm
-UQ==
------END CERTIFICATE-----
- 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC
-   i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root
------BEGIN CERTIFICATE-----
-MIIFVjCCBD6gAwIBAgIQXpDZ0ETJMV02WTx3GTnhhTANBgkqhkiG9w0BAQUFADBv
-MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk
-ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF
-eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDYyNDE5MDYzMFow
-gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl
-bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u
-ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv
-cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg
-Q0EgU0dDMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgOkO3f7wzN6
-rOjg45tR5vjBfzK7qmV9IBxb/QW9EEXxG+E7FNhZqQLtwGBKoSsHTnQqV75wWMk0
-9tinWvftBkSpj5sTi/8cbzJfUvTSVYh3Qxv6AVVjMMH/ruLjE6y+4PoaPs8WoYAQ
-ts5R4Z1g8c/WnTepLst2x0/Wv7GmuoQi+gXvHU6YrBiu7XkeYhzc95QdviWSJRDk
-owhb5K43qhcvjRmBfO/paGlCliDGZp8mHwrI21mwobWpVjTxZRwYO3bd4+TGcI4G
-Ie5wmHwE8F7SK1tgSqbBacKjDa93j7txKkfz/Yd2n7TGqOXiHPsJpG655vrKtnXk
-9vs1zoDeJQIDAQABo4IBljCCAZIwHQYDVR0OBBYEFAdEdoWTKLx/bXjSCuv6TEvf
-2YIfMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMCAGA1UdJQQZ
-MBcGCisGAQQBgjcKAwMGCWCGSAGG+EIEATAYBgNVHSAEETAPMA0GCysGAQQBgOU3
-AgQBMHsGA1UdHwR0MHIwOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0Fk
-ZFRydXN0RXh0ZXJuYWxDQVJvb3QuY3JsMDagNKAyhjBodHRwOi8vY3JsLmNvbW9k
-by5uZXQvQWRkVHJ1c3RFeHRlcm5hbENBUm9vdC5jcmwwgYAGCCsGAQUFBwEBBHQw
-cjA4BggrBgEFBQcwAoYsaHR0cDovL2NydC5jb21vZG9jYS5jb20vQWRkVHJ1c3RV
-VE5TR0NDQS5jcnQwNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQuY29tb2RvLm5ldC9B
-ZGRUcnVzdFVUTlNHQ0NBLmNydDARBglghkgBhvhCAQEEBAMCAgQwDQYJKoZIhvcN
-AQEFBQADggEBAK2zEzs+jcIrVK9oDkdDZNvhuBYTdCfpxfFs+OAujW0bIfJAy232
-euVsnJm6u/+OrqKudD2tad2BbejLLXhMZViaCmK7D9nrXHx4te5EP8rL19SUVqLY
-1pTnv5dhNgEgvA7n5lIzDSYs7yRLsr7HJsYPr6SeYSuZizyX1SNz7ooJ32/F3X98
-RB0Mlc/E0OyOrkQ9/y5IrnpnaSora8CnUrV5XNOg+kyCz9edCyx4D5wXYcwZPVWz
-8aDqquESrezPyjtfi4WRO4s/VD3HLZvOxzMrWAVYCDG9FxaOhF0QGuuG1F7F3GKV
-v6prNyCl016kRl2j1UT+a7gLd8fA25A4C9E=
+cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEdMBsGA1UEAxMUVEJTIFg1MDkg
+Q0EgYnVzaW5lc3MwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDB1PAU
+qudCcz3tmyGcf+u6EkZqonKKHrV4gZYbvVkIRojmmlhfi/jwvpHvo8bqSt/9Rj5S
+jhCDW0pcbI+IPPtD1Jy+CHNSfnMqVDy6CKQ3p5maTzCMG6ZT+XjnvcND5v+FtaiB
+xk1iCX6uvt0jeUtdZvYbyytsSDE6c3Y5//wRxOF8tM1JxibwO3pyER26jbbN2gQz
+m/EkdGjLdJ4svPk23WDAvQ6G0/z2LcAaJB+XLfqRwfQpHQvfKa1uTi8PivC8qtip
+rmNQMMPMjxSK2azX8cKjjTDJiUKaCb4VHlJDWKEsCFRpgJAoAuX8f7Yfs1M4esGo
+sWb3PGspK3O22uIlAgMBAAGjggF6MIIBdjAdBgNVHQ4EFgQUGgkEzPkJj7seoM7U
+pBR/S9UjWM0wDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwGAYD
+VR0gBBEwDzANBgsrBgEEAYDlNwIBATB7BgNVHR8EdDByMDigNqA0hjJodHRwOi8v
+Y3JsLmNvbW9kb2NhLmNvbS9BZGRUcnVzdEV4dGVybmFsQ0FSb290LmNybDA2oDSg
+MoYwaHR0cDovL2NybC5jb21vZG8ubmV0L0FkZFRydXN0RXh0ZXJuYWxDQVJvb3Qu
+Y3JsMIGGBggrBgEFBQcBAQR6MHgwOwYIKwYBBQUHMAKGL2h0dHA6Ly9jcnQuY29t
+b2RvY2EuY29tL0FkZFRydXN0VVROU2VydmVyQ0EuY3J0MDkGCCsGAQUFBzAChi1o
+dHRwOi8vY3J0LmNvbW9kby5uZXQvQWRkVHJ1c3RVVE5TZXJ2ZXJDQS5jcnQwEQYJ
+YIZIAYb4QgEBBAQDAgIEMA0GCSqGSIb3DQEBBQUAA4IBAQA7mqrMgk/MrE6QnbNA
+h4nRCn2ti4bg4w2C3lB6bSvRPnYwuNw9Jb8vuKkNFzRDxNJXqVDZdfFW5CVQJuyd
+nfAx83+wk+spzvFaE1KhFYfN9G9pQfXUfvDRoIcJgPEKUXL1wRiOG+IjU3VVI8pg
+IgqHkr7ylln5i5zCiFAPuIJmYUSFg/gxH5xkCNcjJqqrHrHatJr6Qrrke93joupw
+oU1njfAcZtYp6fbiK6u2b1pJqwkVBE8RsfLnPhRj+SFbpvjv8Od7o/ieJhFIYQNU
+k2jX2u8qZnAiNw93LZW9lpYjtuvMXq8QQppENNja5b53q7UwI+lU7ZGjZ7quuESp
+J6/5
 -----END CERTIFICATE-----
  2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root
-   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC
+   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware
 -----BEGIN CERTIFICATE-----
-MIIEZjCCA06gAwIBAgIQUSYKkxzif5zDpV954HKugjANBgkqhkiG9w0BAQUFADCB
-kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
+MIIETzCCAzegAwIBAgIQHM5EYpUZep1jUvnyI6m2mDANBgkqhkiG9w0BAQUFADCB
+lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
 Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho
-dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw
-IFNHQzAeFw0wNTA2MDcwODA5MTBaFw0xOTA2MjQxOTA2MzBaMG8xCzAJBgNVBAYT
-AlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0
-ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB
-IFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC39xoz5vIABC05
-4E5b7R+8bA/Ntfojts7emxEzl6QpTH2Tn71KvJPtAxrjj8/lbVBa1pcplFqAsEl6
-2y6V/bjKvzc4LR4+kUGtcFbH8E8/6DKedMrIkFTpxl8PeJ2aQDwOrGGqXhSPnoeh
-alDc15pOrwWzpnGUnHGzUGAKxxOdOAeGAqjpqGkmGJCrTLBPI6s6T4TY386f4Wlv
-u9dC12tE5Met7m1BX3JacQg3s3llpFmglDf3AC8NwpJy2tA4ctsUqEXEXSp9t7TW
-xO6szRNEt8kr3UMAJfphuWlqWCMRt6czj1Z1WfXNKddGtworZbbTQm8Vsrh7++/p
-XVPVNFonAgMBAAGjgdgwgdUwHwYDVR0jBBgwFoAUUzLRs89/+uDxoF2FTpLSnkUd
-tE8wHQYDVR0OBBYEFK29mHo0tCb3+sQmVO8DveAky1QaMA4GA1UdDwEB/wQEAwIB
-BjAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBAjAgBgNVHSUEGTAX
-BgorBgEEAYI3CgMDBglghkgBhvhCBAEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDov
-L2NybC51c2VydHJ1c3QuY29tL1VUTi1EQVRBQ29ycFNHQy5jcmwwDQYJKoZIhvcN
-AQEFBQADggEBAMbuUxdoFLJRIh6QWA2U/b3xcOWGLcM2MY9USEbnLQg3vGwKYOEO
-rVE04BKT6b64q7gmtOmWPSiPrmQH/uAB7MXjkesYoPF1ftsK5p+R26+udd8jkWjd
-FwBaS/9kbHDrARrQkNnHptZt9hPk/7XJ0h4qy7ElQyZ42TCbTg0evmnv3+r+LbPM
-+bDdtRTKkdSytaX7ARmjR3mfnYyVhzT4HziS2jamEfpr62vp3EV4FTkG101B5CHI
-3C+H0be/SGB1pWLLJN47YaApIKa+xWycxOkKaSLvkTr6Jq/RW0GnOuL4OAdCq8Fb
-+M5tug8EPzI0rNwEKNdwMBQmBsTkm5jVz3g=
+dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt
+SGFyZHdhcmUwHhcNMDUwNjA3MDgwOTEwWhcNMTkwNzA5MTgxOTIyWjBvMQswCQYD
+VQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0
+IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5h
+bCBDQSBSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/caM+by
+AAQtOeBOW+0fvGwPzbX6I7bO3psRM5ekKUx9k5+9SryT7QMa44/P5W1QWtaXKZRa
+gLBJetsulf24yr83OC0ePpFBrXBWx/BPP+gynnTKyJBU6cZfD3idmkA8Dqxhql4U
+j56HoWpQ3NeaTq8Fs6ZxlJxxs1BgCscTnTgHhgKo6ahpJhiQq0ywTyOrOk+E2N/O
+n+Fpb7vXQtdrROTHre5tQV9yWnEIN7N5ZaRZoJQ39wAvDcKSctrQOHLbFKhFxF0q
+fbe01sTurM0TRLfJK91DACX6YblpalgjEbenM49WdVn1zSnXRrcKK2W200JvFbK4
+e/vv6V1T1TRaJwIDAQABo4G9MIG6MB8GA1UdIwQYMBaAFKFyXyYbKJhDlV0HN9WF
+lp1L0sNFMB0GA1UdDgQWBBStvZh6NLQm9/rEJlTvA73gJMtUGjAOBgNVHQ8BAf8E
+BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhCAQEEBAMCAQIwRAYDVR0f
+BD0wOzA5oDegNYYzaHR0cDovL2NybC51c2VydHJ1c3QuY29tL1VUTi1VU0VSRmly
+c3QtSGFyZHdhcmUuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQByQhANOs4kClrwF8BW
+onvUOGCSjRK52zYZgDXYNjDtmr5rJ6NyPFDNn+JxkLpjYetIFMTbSRe679Bt8m7a
+gIAoQYFQtxMuyLnJegB2aEbQiIxh/tC21UcFF7ktdnDoTlA6w3pLuvunaI84Of3o
+2YBrhzkTbCfaYk5JRlTpudW9DkUkHBsyx3nknPKnplkIGaK0jgn8E0n+SFabYaHk
+I9LroYT/+JtLefh9lgBdAgVv0UPbzoGfuDsrk/Zh+UrgbLFpHoVnElhzbkh64Z0X
+OGaJunQc68cCZu5HTn/aK7fBGMcVflRCXLVEQpU9PIAdGA8Ynvg684t8GMaKsRl1
+jIGZ
 -----END CERTIFICATE-----
- 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC
-   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC
+ 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware
+   i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware
 -----BEGIN CERTIFICATE-----
-MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB
-kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
+MIIEdDCCA1ygAwIBAgIQRL4Mi1AAJLQR0zYq/mUK/TANBgkqhkiG9w0BAQUFADCB
+lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug
 Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho
-dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw
-IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG
-EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD
-VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu
-dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN
-BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6
-E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ
-D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK
-4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq
-lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW
-bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB
-o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT
-MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js
-LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr
-BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB
-AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft
-Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj
-j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH
-KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv
-2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3
-mfnGV/TJVTl4uix5yaaIK/QI
+dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt
+SGFyZHdhcmUwHhcNOTkwNzA5MTgxMDQyWhcNMTkwNzA5MTgxOTIyWjCBlzELMAkG
+A1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0eTEe
+MBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8v
+d3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdh
+cmUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCx98M4P7Sof885glFn
+0G2f0v9Y8+efK+wNiVSZuTiZFvfgIXlIwrthdBKWHTxqctU8EGc6Oe0rE81m65UJ
+M6Rsl7HoxuzBdXmcRl6Nq9Bq/bkqVRcQVLMZ8Jr28bFdtqdt++BxF2uiiPsA3/4a
+MXcMmgF6sTLjKwEHOG7DpV4jvEWbe1DByTCP2+UretNb+zNAHqDVmBe8i4fDidNd
+oI6yqqr2jmmIBsX6iSHzCJ1pLgkzmykNRg+MzEk0sGlRvfkGzWitZky8PqxhvQqI
+DsjfPe58BEydCl5rkdbux+0ojatNh4lz0G6k0B4WixThdkQDf2Os5M1JnMWS9Ksy
+oUhbAgMBAAGjgbkwgbYwCwYDVR0PBAQDAgHGMA8GA1UdEwEB/wQFMAMBAf8wHQYD
+VR0OBBYEFKFyXyYbKJhDlV0HN9WFlp1L0sNFMEQGA1UdHwQ9MDswOaA3oDWGM2h0
+dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VVE4tVVNFUkZpcnN0LUhhcmR3YXJlLmNy
+bDAxBgNVHSUEKjAoBggrBgEFBQcDAQYIKwYBBQUHAwUGCCsGAQUFBwMGBggrBgEF
+BQcDBzANBgkqhkiG9w0BAQUFAAOCAQEARxkP3nTGmZev/K0oXnWO6y1n7k57K9cM
+//bey1WiCuFMVGWTYGufEpytXoMs61quwOQt9ABjHbjAbPLPSbtNk28Gpgoiskli
+CE7/yMgUsogWXecB5BKV5UU0s4tpvc+0hY91UZ59Ojg6FEgSxvunOxqNDYJAB+gE
+CJChicsZUN/KHAG8HQQZexB2lzvukJDKxA4fFm517zP4029bHpbj4HR3dHuKom4t
+3XbWOTCC8KucUvIqx69JXn7HaOWCgchqJ/kniCrVWFCVH/A7HFe7fRQ5YiuayZSS
+KqMiDP+JJn1fIytH1xUdqWqeUQ0qUZ6B+dQ7XnASfxAynB67nfhmqA==
 -----END CERTIFICATE-----
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -111,13 +111,12 @@
         if test_support.verbose:
             sys.stdout.write("\n" + pprint.pformat(p) + "\n")
         self.assertEqual(p['subject'],
-                         ((('countryName', u'US'),),
-                          (('stateOrProvinceName', u'Delaware'),),
-                          (('localityName', u'Wilmington'),),
-                          (('organizationName', u'Python Software Foundation'),),
-                          (('organizationalUnitName', u'SSL'),),
-                          (('commonName', u'somemachine.python.org'),)),
+                         ((('countryName', 'XY'),),
+                          (('localityName', 'Castle Anthrax'),),
+                          (('organizationName', 'Python Software Foundation'),),
+                          (('commonName', 'localhost'),))
                         )
+        self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),))
         # Issue #13034: the subjectAltName in some certificates
         # (notably projects.developer.nokia.com:443) wasn't parsed
         p = ssl._ssl._test_decode_cert(NOKIACERT)
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -363,9 +363,11 @@
     pass
 
 
-def connect(database, **kwargs):
-    factory = kwargs.get("factory", Connection)
-    return factory(database, **kwargs)
+def connect(database, timeout=5.0, detect_types=0, isolation_level="",
+                 check_same_thread=True, factory=None, cached_statements=100):
+    factory = Connection if not factory else factory
+    return factory(database, timeout, detect_types, isolation_level,
+                    check_same_thread, factory, cached_statements)
 
 
 def _unicode_text_factory(x):
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -28,9 +28,11 @@
         return result
 
     elif value.typePtr == typeCache.BooleanType:
-        return result
+        return bool(value.internalRep.longValue)
     elif value.typePtr == typeCache.ByteArrayType:
-        return result
+        size = tkffi.new('int*')
+        data = tklib.Tcl_GetByteArrayFromObj(value, size)
+        return tkffi.buffer(data, size[0])[:]
     elif value.typePtr == typeCache.DoubleType:
         return value.internalRep.doubleValue
     elif value.typePtr == typeCache.IntType:
@@ -50,7 +52,7 @@
             result.append(FromObj(app, tcl_elem[0]))
         return tuple(result)
     elif value.typePtr == typeCache.ProcBodyType:
-        return result
+        pass  # fall through and return tcl object.
     elif value.typePtr == typeCache.StringType:
         buf = tklib.Tcl_GetUnicode(value)
         length = tklib.Tcl_GetCharLength(value)
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -72,6 +72,7 @@
 int Tcl_GetBoolean(Tcl_Interp* interp, const char* src, int* boolPtr);
 char *Tcl_GetString(Tcl_Obj* objPtr);
 char *Tcl_GetStringFromObj(Tcl_Obj* objPtr, int* lengthPtr);
+unsigned char *Tcl_GetByteArrayFromObj(Tcl_Obj* objPtr, int* lengthPtr);
 
 Tcl_UniChar *Tcl_GetUnicode(Tcl_Obj* objPtr);
 int Tcl_GetCharLength(Tcl_Obj* objPtr);
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -10,8 +10,35 @@
 
 import os
 def get_include():
-    head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.join(head, '../include')
+    """
+    Return the directory that contains the NumPy \\*.h header files.
+
+    Extension modules that need to compile against NumPy should use this
+    function to locate the appropriate include directory.
+
+    Notes
+    -----
+    When using ``distutils``, for example in ``setup.py``.
+    ::
+
+        import numpy as np
+        ...
+        Extension('extension_name', ...
+                include_dirs=[np.get_include()])
+        ...
+
+    """
+    import numpy
+    if getattr(numpy, 'show_config', None) is None:
+        # running from numpy source directory
+        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
+        return os.path.join(head, '../include')
+    else:
+        # using installed numpy core headers
+        import numpy.core as core
+        d = os.path.join(os.path.dirname(core.__file__), 'include')
+    return d
+
 
 
 __all__ = ['__version__', 'get_include']
diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -1,1 +1,75 @@
 from _numpypy.numerictypes import *
+import numpypy
+
+def issubclass_(arg1, arg2):
+    """
+    Determine if a class is a subclass of a second class.
+
+    `issubclass_` is equivalent to the Python built-in ``issubclass``,
+    except that it returns False instead of raising a TypeError is one
+    of the arguments is not a class.
+
+    Parameters
+    ----------
+    arg1 : class
+        Input class. True is returned if `arg1` is a subclass of `arg2`.
+    arg2 : class or tuple of classes.
+        Input class. If a tuple of classes, True is returned if `arg1` is a
+        subclass of any of the tuple elements.
+
+    Returns
+    -------
+    out : bool
+        Whether `arg1` is a subclass of `arg2` or not.
+
+    See Also
+    --------
+    issubsctype, issubdtype, issctype
+
+    Examples
+    --------
+    >>> np.issubclass_(np.int32, np.int)
+    True
+    >>> np.issubclass_(np.int32, np.float)
+    False
+
+    """
+    try:
+        return issubclass(arg1, arg2)
+    except TypeError:
+        return False
+
+def issubdtype(arg1, arg2):
+    """
+    Returns True if first argument is a typecode lower/equal in type hierarchy.
+
+    Parameters
+    ----------
+    arg1, arg2 : dtype_like
+        dtype or string representing a typecode.
+
+    Returns
+    -------
+    out : bool
+
+    See Also
+    --------
+    issubsctype, issubclass_
+    numpy.core.numerictypes : Overview of numpy type hierarchy.
+
+    Examples
+    --------
+    >>> np.issubdtype('S1', str)
+    True
+    >>> np.issubdtype(np.float64, np.float32)
+    False
+
+    """
+    if issubclass_(arg2, generic):
+        return issubclass(numpypy.dtype(arg1).type, arg2)
+    mro = numpypy.dtype(arg2).type.mro()
+    if len(mro) > 1:
+        val = mro[1]
+    else:
+        val = mro[0]
+    return issubclass(numpypy.dtype(arg1).type, val)
diff --git a/pypy/TODO b/pypy/TODO
deleted file mode 100644
--- a/pypy/TODO
+++ /dev/null
@@ -1,2 +0,0 @@
-
-* ARM
diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst
--- a/pypy/doc/arm.rst
+++ b/pypy/doc/arm.rst
@@ -35,6 +35,11 @@
   * ``qemu-system``
   * ``qemu-user-static``
 
+- The dependencies above are in addition to the ones needed for a regular
+  translation, `listed here`_.
+
+.. _`listed here`: getting-started-python.html#translating-the-pypy-python-interpreter
+
 
 Creating a Qemu based ARM chroot
 --------------------------------
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -25,6 +25,7 @@
   necessary; also update the version number in pypy/doc/conf.py,
   and in pypy/doc/index.rst
 * update pypy/doc/contributor.rst (and possibly LICENSE)
+  pypy/doc/tool/makecontributor.py generates the list of contributors
 * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
   and create a fresh whatsnew_head.rst after the release
 * update README
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -52,6 +52,10 @@
 .. branch: ndarray-subtype
 Allow subclassing ndarray, i.e. matrix
 
+.. branch: pypy-pyarray
+Implement much of numpy's c api in cpyext, allows (slow) access to ndarray
+from c
+
 .. branch: kill-ootype
 
 .. branch: fast-slowpath
@@ -97,3 +101,5 @@
 Use subclasses of SpaceOperation instead of SpaceOperator objects.
 Random cleanups in flowspace.
 
+.. branch: file-support-in-rpython
+make open() and friends rpython
diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -107,9 +107,15 @@
         for i in range(min(len(varnames), self.getfastscopelength())):
             name = varnames[i]
             w_value = fastscope_w[i]
+            w_name = self.space.wrap(name)
             if w_value is not None:
-                w_name = self.space.wrap(name)
                 self.space.setitem(self.w_locals, w_name, w_value)
+            else:
+                try:
+                    self.space.delitem(self.w_locals, w_name)
+                except OperationError as e:
+                    if not e.match(self.space, self.space.w_KeyError):
+                        raise
 
     def locals2fast(self):
         # Copy values from self.w_locals to the fastlocals
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -954,6 +954,8 @@
 
         sys.path.append(self.goal_dir)
         # make sure cwd does not contain a stdlib
+        if self.tmp_dir.startswith(self.trunkdir):
+            skip('TMPDIR is inside the PyPy source')
         os.chdir(self.tmp_dir)
         tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c')
         try:
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -83,10 +83,21 @@
     def test_locals(self):
         def f():
             return locals()
+
         def g(c=0, b=0, a=0):
             return locals()
+
         assert f() == {}
-        assert g() == {'a':0, 'b':0, 'c':0}
+        assert g() == {'a': 0, 'b': 0, 'c': 0}
+
+    def test_locals_deleted_local(self):
+        def f():
+            a = 3
+            locals()
+            del a
+            return locals()
+
+        assert f() == {}
 
     def test_dir(self):
         def f():
@@ -252,25 +263,9 @@
         assert next(x) == 3
 
     def test_xrange_args(self):
-##        # xrange() attributes are deprecated and were removed in Python 2.3.
-##        x = xrange(2)
-##        assert x.start == 0
-##        assert x.stop == 2
-##        assert x.step == 1
-
-##        x = xrange(2,10,2)
-##        assert x.start == 2
-##        assert x.stop == 10
-##        assert x.step == 2
-
-##        x = xrange(2.3, 10.5, 2.4)
-##        assert x.start == 2
-##        assert x.stop == 10
-##        assert x.step == 2
-
         raises(ValueError, xrange, 0, 1, 0)
 
-    def test_xrange_repr(self): 
+    def test_xrange_repr(self):
         assert repr(xrange(1)) == 'xrange(1)'
         assert repr(xrange(1,2)) == 'xrange(1, 2)'
         assert repr(xrange(1,2,3)) == 'xrange(1, 4, 3)'
@@ -329,7 +324,7 @@
         raises(TypeError, xrange, 1, 3+2j)
         raises(TypeError, xrange, 1, 2, '1')
         raises(TypeError, xrange, 1, 2, 3+2j)
-    
+
     def test_sorted(self):
         l = []
         sorted_l = sorted(l)
@@ -348,7 +343,7 @@
         assert sorted_l is not l
         assert sorted_l == ['C', 'b', 'a']
         raises(TypeError, sorted, [], reverse=None)
-        
+
     def test_reversed_simple_sequences(self):
         l = range(5)
         rev = reversed(l)
@@ -364,8 +359,8 @@
                 return 42
         obj = SomeClass()
         assert reversed(obj) == 42
-    
-        
+
+
     def test_cmp(self):
         assert cmp(9,9) == 0
         assert cmp(0,9) < 0
@@ -398,7 +393,7 @@
         raises(RuntimeError, cmp, a, c)
         # okay, now break the cycles
         a.pop(); b.pop(); c.pop()
-        
+
     def test_coerce(self):
         assert coerce(1, 2)    == (1, 2)
         assert coerce(1L, 2L)  == (1L, 2L)
@@ -465,7 +460,7 @@
         assert eval("1+2") == 3
         assert eval(" \t1+2\n") == 3
         assert eval("len([])") == 0
-        assert eval("len([])", {}) == 0        
+        assert eval("len([])", {}) == 0
         # cpython 2.4 allows this (raises in 2.3)
         assert eval("3", None, None) == 3
         i = 4
@@ -683,15 +678,15 @@
         w_value = space.getitem(w_dict, space.wrap('i'))
         assert space.eq_w(w_value, space.wrap(42))
 
-    def test_execfile_different_lineendings(self, space): 
+    def test_execfile_different_lineendings(self, space):
         from rpython.tool.udir import udir
         d = udir.ensure('lineending', dir=1)
-        dos = d.join('dos.py') 
-        f = dos.open('wb') 
+        dos = d.join('dos.py')
+        f = dos.open('wb')
         f.write("x=3\r\n\r\ny=4\r\n")
-        f.close() 
+        f.close()
         space.appexec([space.wrap(str(dos))], """
-            (filename): 
+            (filename):
                 d = {}
                 execfile(filename, d)
                 assert d['x'] == 3
@@ -699,12 +694,12 @@
         """)
 
         unix = d.join('unix.py')
-        f = unix.open('wb') 
+        f = unix.open('wb')
         f.write("x=5\n\ny=6\n")
-        f.close() 
+        f.close()
 
         space.appexec([space.wrap(str(unix))], """
-            (filename): 
+            (filename):
                 d = {}
                 execfile(filename, d)
                 assert d['x'] == 5
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
-import py
+import py, sys
 from pypy.module._pypyjson.interp_decoder import JSONDecoder
 
 def test_skip_whitespace():
@@ -16,6 +16,9 @@
 class AppTest(object):
     spaceconfig = {"objspace.usemodules._pypyjson": True}
 
+    def setup_class(cls):
+        cls.w_run_on_16bit = cls.space.wrap(sys.maxunicode == 65535)
+
     def test_raise_on_unicode(self):
         import _pypyjson
         raises(TypeError, _pypyjson.loads, u"42")
@@ -178,11 +181,11 @@
         raises(ValueError, "_pypyjson.loads('[1: 2]')")
         raises(ValueError, "_pypyjson.loads('[1, 2')")
         raises(ValueError, """_pypyjson.loads('["extra comma",]')""")
-        
+
     def test_unicode_surrogate_pair(self):
+        if self.run_on_16bit:
+            skip("XXX fix me or mark definitely skipped")
         import _pypyjson
         expected = u'z\U0001d120x'
         res = _pypyjson.loads('"z\\ud834\\udd20x"')
         assert res == expected
-
-
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -400,6 +400,8 @@
     def test_socket_connect_ex(self):
         import _socket
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+        # The following might fail if the DNS redirects failed requests to a
+        # catch-all address (i.e. opendns).
         # Make sure we get an app-level error, not an interp one.
         raises(_socket.gaierror, s.connect_ex, ("wrong.invalid", 80))
         s.close()
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -399,7 +399,7 @@
 
         proto = libssl_SSL_CIPHER_get_version(current)
         if proto:
-            w_proto = space.wrap(rffi.charp2str(name))
+            w_proto = space.wrap(rffi.charp2str(proto))
         else:
             w_proto = space.w_None
 
@@ -476,15 +476,15 @@
                 w_serial = space.wrap(rffi.charpsize2str(buf, length))
             space.setitem(w_retval, space.wrap("serialNumber"), w_serial)
 
-            libssl_BIO_reset(biobuf)
-            notBefore = libssl_X509_get_notBefore(certificate)
-            libssl_ASN1_TIME_print(biobuf, notBefore)
-            with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as buf:
-                length = libssl_BIO_gets(biobuf, buf, 99)
-                if length < 0:
-                    raise _ssl_seterror(space, None, length)
-                w_date = space.wrap(rffi.charpsize2str(buf, length))
-            space.setitem(w_retval, space.wrap("notBefore"), w_date)
+        libssl_BIO_reset(biobuf)
+        notBefore = libssl_X509_get_notBefore(certificate)
+        libssl_ASN1_TIME_print(biobuf, notBefore)
+        with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as buf:
+            length = libssl_BIO_gets(biobuf, buf, 99)
+            if length < 0:
+                raise _ssl_seterror(space, None, length)
+            w_date = space.wrap(rffi.charpsize2str(buf, length))
+        space.setitem(w_retval, space.wrap("notBefore"), w_date)
 
         libssl_BIO_reset(biobuf)
         notAfter = libssl_X509_get_notAfter(certificate)
@@ -733,7 +733,6 @@
         # Set both the read and write BIO's to non-blocking mode
         libssl_BIO_set_nbio(libssl_SSL_get_rbio(ss.ssl), 1)
         libssl_BIO_set_nbio(libssl_SSL_get_wbio(ss.ssl), 1)
-    libssl_SSL_set_connect_state(ss.ssl)
 
     if side == PY_SSL_CLIENT:
         libssl_SSL_set_connect_state(ss.ssl)
diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -4,11 +4,11 @@
 """
 
 from pypy.interpreter.error import OperationError
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import cpython_api, Py_ssize_t, CANNOT_FAIL
-from pypy.module.cpyext.pyobject import PyObject
-from pypy.module.micronumpy.interp_numarray import W_NDimArray, convert_to_array, wrap_impl
-from pypy.module.micronumpy.interp_dtype import get_dtype_cache
+from pypy.module.cpyext.api import PyObject
+from pypy.module.micronumpy.interp_numarray import W_NDimArray, array
+from pypy.module.micronumpy.interp_dtype import get_dtype_cache, W_Dtype
 from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArray
 from pypy.module.micronumpy.arrayimpl.scalar import Scalar
 from rpython.rlib.rawstorage import RAW_STORAGE_PTR
@@ -113,10 +113,12 @@
     assert isinstance(w_array, W_NDimArray)
     return rffi.cast(rffi.VOIDP, w_array.implementation.storage)
 
+PyArray_Descr = PyObject
+NULL = lltype.nullptr(rffi.VOIDP.TO)
 
- at cpython_api([PyObject, rffi.VOIDP, Py_ssize_t, Py_ssize_t, Py_ssize_t, rffi.VOIDP],
+ at cpython_api([PyObject, PyArray_Descr, Py_ssize_t, Py_ssize_t, Py_ssize_t, rffi.VOIDP],
              PyObject)
-def _PyArray_FromAny(space, w_obj, dtype, min_depth, max_depth, requirements, context):
+def _PyArray_FromAny(space, w_obj, w_dtype, min_depth, max_depth, requirements, context):
     """ This is the main function used to obtain an array from any nested
          sequence, or object that exposes the array interface, op. The
          parameters allow specification of the required dtype, the
@@ -147,17 +149,17 @@
          only used if the array is constructed that way. Almost always this
          parameter is NULL.
     """
-    if dtype:
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            '_PyArray_FromAny called with not-implemented dtype argument'))
-    if min_depth !=0 or max_depth != 0:
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            '_PyArray_FromAny called with not-implemented min_dpeth or max_depth argument'))
     if requirements not in (0, NPY_DEFAULT):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             '_PyArray_FromAny called with not-implemented requirements argument'))
-    w_array = convert_to_array(space, w_obj)
-    if w_array.is_scalar():
+    w_array = array(space, w_obj, w_dtype=w_dtype, copy=False)
+    if min_depth !=0 and len(w_array.get_shape()) < min_depth:
+        raise OperationError(space.w_ValueError, space.wrap(
+            'object of too small depth for desired array'))
+    elif max_depth !=0 and len(w_array.get_shape()) > max_depth:
+        raise OperationError(space.w_ValueError, space.wrap(
+            'object of too deep for desired array'))
+    elif w_array.is_scalar():
         # since PyArray_DATA() fails on scalars, create a 1D array and set empty
         # shape. So the following combination works for *reading* scalars:
         #     PyObject *arr = PyArray_FromAny(obj);
@@ -171,8 +173,21 @@
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, Py_ssize_t], PyObject)
 def _PyArray_FromObject(space, w_obj, typenum, min_depth, max_depth):
-    return _PyArray_FromAny(space, w_obj, typenum, min_depth, max_depth, NPY_BEHAVED);
-
+    try:
+        dtype = get_dtype_cache(space).dtypes_by_num[typenum]
+    except KeyError:
+        raise OperationError(space.w_ValueError, space.wrap(
+            '_PyArray_FromObject called with invalid dtype %d' % typenum))
+    try:
+        return _PyArray_FromAny(space, w_obj, dtype, min_depth, max_depth,
+                            0, NULL);
+    except OperationError, e:
+        if e.match(space, space.w_NotImplementedError):
+            errstr = space.str_w(e.get_w_value(space))
+            errstr = '_PyArray_FromObject' + errstr[16:]
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                errstr))
+        raise
 
 def get_shape_and_dtype(space, nd, dims, typenum):
     shape = []
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -103,11 +103,13 @@
     exc_p[0] = make_ref(space, operr.w_type)
     val_p[0] = make_ref(space, operr.get_w_value(space))
 
- at cpython_api([], lltype.Void)
+ at cpython_api([], rffi.INT_real, error=0)
 def PyErr_BadArgument(space):
     """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where
     message indicates that a built-in operation was invoked with an illegal
-    argument.  It is mostly for internal use."""
+    argument.  It is mostly for internal use. In CPython this function always
+    raises an exception and returns 0 in all cases, hence the (ab)use of the
+    error indicator."""
     raise OperationError(space.w_TypeError,
             space.wrap("bad argument type for built-in operation"))
 
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -46,7 +46,7 @@
             raise Exception("DID NOT RAISE")
         if getattr(space, 'w_' + expected_exc.__name__) is not operror.w_type:
             raise Exception("Wrong exception")
-        state.clear_exception()
+        return state.clear_exception()
 
     def setup_method(self, func):
         freeze_refcnts(self)
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -90,16 +90,16 @@
     def test_FromAny(self, space, api):
         a = array(space, [10, 5, 3])
         assert api._PyArray_FromAny(a, NULL, 0, 0, 0, NULL) is a
-        self.raises(space, api, NotImplementedError, api._PyArray_FromAny,
-                    space.wrap(a), space.w_None, space.wrap(0),
-                    space.wrap(3), space.wrap(0), space.w_None)
+        assert api._PyArray_FromAny(a, NULL, 1, 4, 0, NULL) is a
+        self.raises(space, api, ValueError, api._PyArray_FromAny,
+                    a, NULL, 4, 5, 0, NULL)
 
     def test_FromObject(self, space, api):
         a = array(space, [10, 5, 3])
-        assert api._PyArray_FromObject(a, NULL, 0, 0, 0, NULL) is a
-        self.raises(space, api, NotImplementedError, api._PyArray_FromObject,
-                    space.wrap(a), space.w_None, space.wrap(0),
-                    space.wrap(3), space.wrap(0), space.w_None)
+        assert api._PyArray_FromObject(a, a.get_dtype().num, 0, 0) is a
+        exc = self.raises(space, api, ValueError, api._PyArray_FromObject,
+                    a, 11, 4, 5)
+        assert exc.errorstr(space).find('desired') >= 0
 
     def test_list_from_fixedptr(self, space, api):
         A = lltype.GcArray(lltype.Float)
@@ -241,6 +241,27 @@
                 PyObject * obj2 = PyArray_ZEROS(2, dims2, 11, 0);
                 PyArray_FILLWBYTE(obj2, 42);
                 PyArray_CopyInto(obj2, obj1);
+                Py_DECREF(obj1);
+                return obj2;
+                '''
+                ),
+                ("test_FromAny", "METH_NOARGS",
+                '''
+                npy_intp dims[2] ={2, 3};
+                PyObject * obj1 = PyArray_SimpleNew(2, dims, 1);
+                PyArray_FILLWBYTE(obj1, 42);
+                PyObject * obj2 = _PyArray_FromAny(obj1, NULL, 0, 0, 0, NULL);
+                Py_DECREF(obj1);
+                return obj2;
+                '''
+                ),
+                 ("test_FromObject", "METH_NOARGS",
+                '''
+                npy_intp dims[2] ={2, 3};
+                PyObject * obj1 = PyArray_SimpleNew(2, dims, 1);
+                PyArray_FILLWBYTE(obj1, 42);
+                PyObject * obj2 = _PyArray_FromObject(obj1, 12, 0, 0);
+                Py_DECREF(obj1);
                 return obj2;
                 '''
                 ),
@@ -254,3 +275,6 @@
         assert (arr == 42).all()
         arr = mod.test_copy()
         assert (arr == 0).all()
+        #Make sure these work without errors
+        arr = mod.test_FromAny()
+        arr = mod.test_FromObject()
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -70,9 +70,10 @@
         api.PyErr_Clear()
 
     def test_BadArgument(self, space, api):
-        api.PyErr_BadArgument()
+        ret = api.PyErr_BadArgument()
         state = space.fromcache(State)
         assert space.eq_w(state.operror.w_type, space.w_TypeError)
+        assert ret == 0
         api.PyErr_Clear()
 
     def test_Warning(self, space, api, capfd):
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -91,6 +91,7 @@
         invalid = rffi.str2charp('invalid')
         utf_8 = rffi.str2charp('utf-8')
         prev_encoding = rffi.str2charp(space.unwrap(w_default_encoding))
+        self.raises(space, api, TypeError, api.PyUnicode_SetDefaultEncoding, lltype.nullptr(rffi.CCHARP.TO))
         assert api.PyUnicode_SetDefaultEncoding(invalid) == -1
         assert api.PyErr_Occurred() is space.w_LookupError
         api.PyErr_Clear()
@@ -316,6 +317,15 @@
         rffi.free_charp(b_text)
         rffi.free_charp(b_encoding)
 
+    def test_decode_null_encoding(self, space, api):
+        null_charp = lltype.nullptr(rffi.CCHARP.TO)
+        u_text = u'abcdefg'
+        s_text = space.str_w(api.PyUnicode_AsEncodedString(space.wrap(u_text), null_charp, null_charp))
+        b_text = rffi.str2charp(s_text)
+        assert space.unwrap(api.PyUnicode_Decode(b_text, len(s_text), null_charp, null_charp)) == u_text
+        self.raises(space, api, TypeError, api.PyUnicode_FromEncodedObject, space.wrap(u_text), null_charp, None)
+        rffi.free_charp(b_text)
+
     def test_leak(self):
         size = 50
         raw_buf, gc_buf = rffi.alloc_buffer(size)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -273,6 +273,8 @@
 def PyUnicode_SetDefaultEncoding(space, encoding):
     """Sets the currently active default encoding. Returns 0 on
     success, -1 in case of an error."""
+    if not encoding:
+        PyErr_BadArgument(space)
     w_encoding = space.wrap(rffi.charp2str(encoding))
     setdefaultencoding(space, w_encoding)
     default_encoding[0] = '\x00'
@@ -350,8 +352,11 @@
     in the unicode() built-in function.  The codec to be used is looked up
     using the Python codec registry.  Return NULL if an exception was raised by
     the codec."""
+    if not encoding:
+        # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead
+        encoding = PyUnicode_GetDefaultEncoding(space)
+    w_encoding = space.wrap(rffi.charp2str(encoding))
     w_str = space.wrap(rffi.charpsize2str(s, size))
-    w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))
     else:
@@ -379,6 +384,9 @@
 
     All other objects, including Unicode objects, cause a TypeError to be
     set."""
+    if not encoding:
+        raise OperationError(space.w_TypeError,
+                             space.wrap("decoding Unicode is not supported"))
     w_encoding = space.wrap(rffi.charp2str(encoding))
     if errors:
         w_errors = space.wrap(rffi.charp2str(errors))
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -334,9 +334,8 @@
             v = hi
         del partials[added:]
         if v != 0.0:
-            if rfloat.isinf(v) or rfloat.isnan(v):
-                if (not rfloat.isinf(original) and
-                    not rfloat.isnan(original)):
+            if not rfloat.isfinite(v):
+                if rfloat.isfinite(original):
                     raise OperationError(space.w_OverflowError,
                                          space.wrap("intermediate overflow"))
                 if rfloat.isinf(original):
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -86,7 +86,7 @@
         return W_NDimArray(scalar.Scalar(dtype, w_val))
 
 
-def convert_to_array(space, w_obj, use_prepare=False):
+def convert_to_array(space, w_obj):
     #XXX: This whole routine should very likely simply be array()
     from pypy.module.micronumpy.interp_numarray import array
     from pypy.module.micronumpy import interp_ufuncs
@@ -101,7 +101,7 @@
             if isinstance(w_result, W_NDimArray):
                 return w_result
             else:
-                raise OperationError(space.w_ValueError,
+                raise OperationError(space.w_ValueError, 
                         space.wrap("object __array__ method not producing an array"))
         elif issequence_w(space, w_obj):
             # Convert to array.
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -357,7 +357,16 @@
         elif name == 'logical_or':
             self.done_func = done_if_true
         else:
-            self.done_func = None
+            self.done_func = Non
+
+    def are_common_types(self, dtype1, dtype2):
+        if dtype1.is_complex_type() and dtype2.is_complex_type():
+            return True
+        elif not (dtype1.is_complex_type() or dtype2.is_complex_type()) and \
+                (dtype1.is_int_type() and dtype2.is_int_type() or dtype1.is_float_type() and dtype2.is_float_type()) and \
+                not (dtype1.is_bool_type() or dtype2.is_bool_type()):
+            return True
+        return False
 
     @jit.unroll_safe
     def call(self, space, args_w):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2967,6 +2967,17 @@
 
         assert len(list(a[0])) == 2
 
+    def test_issue_1589(self):
+        import numpypy as numpy
+        c = numpy.array([[(1, 2, 'a'), (3, 4, 'b')], [(5, 6, 'c'), (7, 8, 'd')]],
+                        dtype=[('bg', 'i8'), ('fg', 'i8'), ('char', 'S1')])
+        assert c[0][0]["char"] == 'a'
+
+    def test_scalar_coercion(self):
+        import numpypy as np
+        a = np.array([1,2,3], dtype=np.int16)
+        assert (a * 2).dtype == np.int16
+
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
diff --git a/pypy/module/micronumpy/tool/numready/page.html b/pypy/module/micronumpy/tool/numready/page.html
--- a/pypy/module/micronumpy/tool/numready/page.html
+++ b/pypy/module/micronumpy/tool/numready/page.html
@@ -3,9 +3,11 @@
     <head>
         <title>NumPyPy Status</title>
         <meta http-equiv="content-type" content="text/html; charset=utf-8">
+        <meta name="description" content="How much of numpy exists in pypy?" >
         <style type="text/css">
             body {
                 font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace;
+                border: 8px
             }
             h1 {
                 text-align: center;
@@ -33,8 +35,10 @@
         </style>
     </head>
     <body>
-        <h1>NumPyPy Status</h1>
+        <h1>NumPyPy Status: how much of numpy can you use in pypy?</h1>
         <h3>Version: {{ ver }}</h3>
+        <h3>numpy compatability test results, generated automatically by running</br>
+        <code>pypy/module/micronumpy/tool/numready/main.py &lt;path-to-latest-pypy&gt;</code></h3>
         <h3>Overall: {{ msg }}</h3>
         <table>
             <thead>
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1766,14 +1766,14 @@
 
     def store(self, arr, i, offset, box):
         assert isinstance(box, interp_boxes.W_StringBox)
-        # XXX simplify to range(box.dtype.get_size()) ?
         return self._store(arr.storage, i, offset, box)
 
     @jit.unroll_safe
     def _store(self, storage, i, offset, box):
         assert isinstance(box, interp_boxes.W_StringBox)
-        for k in range(min(self.size, box.arr.size-offset)):
-            storage[k + i] = box.arr.storage[k + offset]
+        # XXX simplify to range(box.dtype.get_size()) ?
+        for k in range(min(self.size, box.arr.size-box.ofs)):
+            storage[k + offset + i] = box.arr.storage[k + box.ofs]
 
     def read(self, arr, i, offset, dtype=None):
         if dtype is None:
diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py
--- a/pypy/module/operator/__init__.py
+++ b/pypy/module/operator/__init__.py
@@ -50,6 +50,7 @@
         '__concat__' : 'concat',
         '__contains__' : 'contains',
         'sequenceIncludes' : 'contains',
+        '__index__' : 'index',
         '__delitem__' : 'delitem',
         '__div__' : 'div',
         '__eq__' : 'eq',
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -190,3 +190,9 @@
         assert methodcaller("method", 4)(x) == (4, 3)
         assert methodcaller("method", 4, 5)(x) == (4, 5)
         assert methodcaller("method", 4, arg2=42)(x) == (4, 42)
+
+    def test_index(self):
+        import operator
+        assert operator.index(42) == 42
+        assert operator.__index__(42) == 42
+        raises(TypeError, operator.index, "abc")
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -149,6 +149,8 @@
         interpleveldefs['nice'] = 'interp_posix.nice'
     if hasattr(os, 'getlogin'):
         interpleveldefs['getlogin'] = 'interp_posix.getlogin'
+    if hasattr(os, 'ctermid'):
+        interpleveldefs['ctermid'] = 'interp_posix.ctermid'
 
     for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid',
                  'seteuid', 'setgid', 'setegid', 'getgroups', 'getpgrp',
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1218,3 +1218,10 @@
         return space.wrap(rurandom.urandom(context, n))
     except OSError, e:
         raise wrap_oserror(space, e)
+
+def ctermid(space):
+    """ctermid() -> string
+
+    Return the name of the controlling terminal for this process.
+    """
+    return space.wrap(os.ctermid())
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -110,7 +110,7 @@
                        '__pypy__', 'cStringIO', '_collections', 'struct',
                        'mmap', 'marshal', '_codecs', 'rctime', 'cppyy',
                        '_cffi_backend', 'pyexpat', '_continuation', '_io',
-                       'thread']:
+                       'thread', 'select']:
             if modname == 'pypyjit' and 'interp_resop' in rest:
                 return False
             return True
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -49,12 +49,15 @@
     from pypy.module.thread.os_lock import Lock
     assert pypypolicy.look_inside_function(Lock.descr_lock_acquire.im_func)
 
+def test_select():
+    from pypy.module.select.interp_select import poll
+    assert pypypolicy.look_inside_function(poll)
+
 def test_pypy_module():
     from pypy.module._collections.interp_deque import W_Deque
     from pypy.module._random.interp_random import W_Random
     assert not pypypolicy.look_inside_function(W_Random.random)
     assert pypypolicy.look_inside_function(W_Deque.length)
-    assert not pypypolicy.look_inside_pypy_module('select.interp_epoll')
     assert pypypolicy.look_inside_pypy_module('__builtin__.operation')
     assert pypypolicy.look_inside_pypy_module('__builtin__.abstractinst')
     assert pypypolicy.look_inside_pypy_module('__builtin__.functional')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -277,3 +277,28 @@
             f1 = call_release_gil(..., descr=<Calli 4 ii EF=6 OS=62>)
             ...
         """)
+
+    def test__cffi_bug1(self):
+        from rpython.rlib.test.test_clibffi import get_libm_name
+        def main(libm_name):
+            try:
+                import _cffi_backend
+            except ImportError:
+                sys.stderr.write('SKIP: cannot import _cffi_backend\n')
+                return 0
+
+            libm = _cffi_backend.load_library(libm_name)
+            BDouble = _cffi_backend.new_primitive_type("double")
+            BSin = _cffi_backend.new_function_type([BDouble], BDouble)
+            sin = libm.load_function(BSin, 'sin')
+
+            def f(*args):
+                for i in range(300):
+                    sin(*args)
+
+            f(1.0)
+            f(1)
+        #
+        libm_name = get_libm_name(sys.platform)
+        log = self.run(main, [libm_name])
+        # assert did not crash
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -113,7 +113,7 @@
             i13 = strgetitem(p9, 0)
             i15 = int_eq(i13, 45)
             guard_false(i15, descr=...)
-            i17 = int_sub(0, i10)
+            i17 = int_neg(i10)
             i19 = int_gt(i10, 23)
             guard_false(i19, descr=...)
             p21 = newstr(23)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -60,35 +60,30 @@
         assert log.result == main(500)
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-        i58 = int_gt(i43, 0)
-        guard_true(i58, descr=<Guard0x10483adb8>)
-        p59 = getfield_gc(p15, descr=<FieldP pypy.module.thread.os_lock.Lock.inst_lock 8>)
-        i60 = getfield_gc(p59, descr=<FieldU rpython.rlib.rthread.Lock.inst__lock 8>)
+        i55 = int_gt(i43, 0)
+        guard_true(i55, descr=...)
+        p56 = force_token()
+        setfield_gc(p0, p56, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        i57 = call_release_gil(..., i36, 1, descr=<Calli 4 ii EF=6>)
+        guard_not_forced(descr=...)
+        guard_no_exception(descr=...)
+        i58 = int_is_true(i57)
+        guard_true(i58, descr=...)
+        i59 = int_sub(i43, 1)
+        guard_not_invalidated(descr=...)
         p61 = force_token()
-        setfield_gc(p0, p61, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 24>)
-        i62 = call_release_gil(4312440032, i60, 1, descr=<Calli 4 ii EF=6>)
-        guard_not_forced(descr=<Guard0x103f3cca0>)
-        guard_no_exception(descr=<Guard0x10483ad40>)
+        setfield_gc(p0, p61, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        i62 = call_release_gil(..., i36, 0, descr=<Calli 4 ii EF=6>)
+        guard_not_forced(descr=...)
+        guard_no_exception(descr=...)
         i63 = int_is_true(i62)
-        guard_true(i63, descr=<Guard0x10483acc8>)
-        i64 = int_sub(i43, 1)
-        guard_not_invalidated(descr=<Guard0x10483ac50>)
-        p66 = getfield_gc(p15, descr=<FieldP pypy.module.thread.os_lock.Lock.inst_lock 8>)
-        i67 = getfield_gc(p66, descr=<FieldU rpython.rlib.rthread.Lock.inst__lock 8>)
-        p68 = force_token()
-        setfield_gc(p0, p68, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 24>)
-        i69 = call_release_gil(4312440032, i67, 0, descr=<Calli 4 ii EF=6>)
-        guard_not_forced(descr=<Guard0x103f3cc20>)
-        guard_no_exception(descr=<Guard0x10483aae8>)
-        i70 = int_is_true(i69)
-        guard_false(i70, descr=<Guard0x10483aa70>)
-        i71 = getfield_gc(p66, descr=<FieldU rpython.rlib.rthread.Lock.inst__lock 8>)
-        p72 = force_token()
-        setfield_gc(p0, p72, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 24>)
-        call_release_gil(4312441056, i71, descr=<Callv 0 i EF=6>)
-        guard_not_forced(descr=<Guard0x103f3cba0>)
-        guard_no_exception(descr=<Guard0x10483a9f8>)
-        guard_not_invalidated(descr=<Guard0x10483a980>)
+        guard_false(i63, descr=...)
+        p64 = force_token()
+        setfield_gc(p0, p64, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        call_release_gil(..., i36, descr=<Callv 0 i EF=6>)
+        guard_not_forced(descr=...)
+        guard_no_exception(descr=...)
+        guard_not_invalidated(descr=...)
         --TICK--
-        jump(..., descr=TargetToken(4361239720))
+        jump(..., descr=...)
         """)
diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -180,12 +180,12 @@
                         i = 0
                         for w_ev in space.listview(w_changelist):
                             ev = space.interp_w(W_Kevent, w_ev)
-                            changelist[i].c_ident = ev.event.c_ident
-                            changelist[i].c_filter = ev.event.c_filter
-                            changelist[i].c_flags = ev.event.c_flags
-                            changelist[i].c_fflags = ev.event.c_fflags
-                            changelist[i].c_data = ev.event.c_data
-                            changelist[i].c_udata = ev.event.c_udata
+                            changelist[i].c_ident = ev.ident
+                            changelist[i].c_filter = ev.filter
+                            changelist[i].c_flags = ev.flags
+                            changelist[i].c_fflags = ev.fflags
+                            changelist[i].c_data = ev.data
+                            changelist[i].c_udata = ev.udata
                             i += 1
                         pchangelist = changelist
                     else:
@@ -206,13 +206,12 @@
                             evt = eventlist[i]
 
                             w_event = W_Kevent(space)
-                            w_event.event = lltype.malloc(kevent, flavor="raw")
-                            w_event.event.c_ident = evt.c_ident
-                            w_event.event.c_filter = evt.c_filter
-                            w_event.event.c_flags = evt.c_flags
-                            w_event.event.c_fflags = evt.c_fflags
-                            w_event.event.c_data = evt.c_data
-                            w_event.event.c_udata = evt.c_udata
+                            w_event.ident = evt.c_ident
+                            w_event.filter = evt.c_filter
+                            w_event.flags = evt.c_flags
+                            w_event.fflags = evt.c_fflags
+                            w_event.data = evt.c_data
+                            w_event.udata = evt.c_udata
 
                             elist_w[i] = w_event
 
@@ -234,11 +233,12 @@
 
 class W_Kevent(W_Root):
     def __init__(self, space):
-        self.event = lltype.nullptr(kevent)
-
-    def __del__(self):
-        if self.event:
-            lltype.free(self.event, flavor="raw")
+        self.ident = rffi.cast(kevent.c_ident, 0)
+        self.filter = rffi.cast(kevent.c_filter, 0)
+        self.flags = rffi.cast(kevent.c_flags, 0)
+        self.fflags = rffi.cast(kevent.c_fflags, 0)
+        self.data = rffi.cast(kevent.c_data, 0)
+        self.udata = lltype.nullptr(rffi.VOIDP.TO)
 
     @unwrap_spec(filter=int, flags='c_uint', fflags='c_uint', data=int, udata=r_uint)
     def descr__init__(self, space, w_ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=r_uint(0)):
@@ -247,35 +247,34 @@
         else:
             ident = r_uint(space.c_filedescriptor_w(w_ident))
 
-        self.event = lltype.malloc(kevent, flavor="raw")
-        rffi.setintfield(self.event, "c_ident", ident)
-        rffi.setintfield(self.event, "c_filter", filter)
-        rffi.setintfield(self.event, "c_flags", flags)
-        rffi.setintfield(self.event, "c_fflags", fflags)
-        rffi.setintfield(self.event, "c_data", data)
-        self.event.c_udata = rffi.cast(rffi.VOIDP, udata)
+        self.ident = rffi.cast(kevent.c_ident, ident)
+        self.filter = rffi.cast(kevent.c_filter, filter)
+        self.flags = rffi.cast(kevent.c_flags, flags)
+        self.fflags = rffi.cast(kevent.c_fflags, fflags)
+        self.data = rffi.cast(kevent.c_data, data)
+        self.udata = rffi.cast(rffi.VOIDP, udata)
 
     def _compare_all_fields(self, other, op):
         if IDENT_UINT:
-            l_ident = rffi.cast(lltype.Unsigned, self.event.c_ident)
-            r_ident = rffi.cast(lltype.Unsigned, other.event.c_ident)
+            l_ident = rffi.cast(lltype.Unsigned, self.ident)
+            r_ident = rffi.cast(lltype.Unsigned, other.ident)
         else:
-            l_ident = self.event.c_ident
-            r_ident = other.event.c_ident
-        l_filter = rffi.cast(lltype.Signed, self.event.c_filter)
-        r_filter = rffi.cast(lltype.Signed, other.event.c_filter)
-        l_flags = rffi.cast(lltype.Unsigned, self.event.c_flags)
-        r_flags = rffi.cast(lltype.Unsigned, other.event.c_flags)
-        l_fflags = rffi.cast(lltype.Unsigned, self.event.c_fflags)
-        r_fflags = rffi.cast(lltype.Unsigned, other.event.c_fflags)
+            l_ident = self.ident
+            r_ident = other.ident
+        l_filter = rffi.cast(lltype.Signed, self.filter)
+        r_filter = rffi.cast(lltype.Signed, other.filter)
+        l_flags = rffi.cast(lltype.Unsigned, self.flags)
+        r_flags = rffi.cast(lltype.Unsigned, other.flags)
+        l_fflags = rffi.cast(lltype.Unsigned, self.fflags)
+        r_fflags = rffi.cast(lltype.Unsigned, other.fflags)
         if IDENT_UINT:
-            l_data = rffi.cast(lltype.Signed, self.event.c_data)
-            r_data = rffi.cast(lltype.Signed, other.event.c_data)
+            l_data = rffi.cast(lltype.Signed, self.data)
+            r_data = rffi.cast(lltype.Signed, other.data)
         else:
-            l_data = self.event.c_data
-            r_data = other.event.c_data
-        l_udata = rffi.cast(lltype.Unsigned, self.event.c_udata)
-        r_udata = rffi.cast(lltype.Unsigned, other.event.c_udata)
+            l_data = self.data
+            r_data = other.data
+        l_udata = rffi.cast(lltype.Unsigned, self.udata)
+        r_udata = rffi.cast(lltype.Unsigned, other.udata)
 
         if op == "eq":
             return l_ident == r_ident and \
@@ -330,22 +329,22 @@
         return space.wrap(self.compare_all_fields(space, w_other, "gt"))
 
     def descr_get_ident(self, space):
-        return space.wrap(self.event.c_ident)
+        return space.wrap(self.ident)
 
     def descr_get_filter(self, space):
-        return space.wrap(self.event.c_filter)
+        return space.wrap(self.filter)
 
     def descr_get_flags(self, space):
-        return space.wrap(self.event.c_flags)
+        return space.wrap(self.flags)
 
     def descr_get_fflags(self, space):
-        return space.wrap(self.event.c_fflags)
+        return space.wrap(self.fflags)
 
     def descr_get_data(self, space):
-        return space.wrap(self.event.c_data)
+        return space.wrap(self.data)
 
     def descr_get_udata(self, space):
-        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.event.c_udata))
+        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.udata))
 
 
 W_Kevent.typedef = TypeDef("select.kevent",
diff --git a/pypy/module/test_lib_pypy/numpypy/test_numpy.py b/pypy/module/test_lib_pypy/numpypy/test_numpy.py
--- a/pypy/module/test_lib_pypy/numpypy/test_numpy.py
+++ b/pypy/module/test_lib_pypy/numpypy/test_numpy.py
@@ -96,10 +96,13 @@
 
     def test___all__(self):
         import numpy
-        assert '__all__' in numpy
+        assert '__all__' in dir(numpy)
         assert 'numpypy' not in dir(numpy)
 
     def test_get_include(self):
+        import sys
+        if not hasattr(sys, 'pypy_translation_info'):
+            skip("pypy white-box test")
         import numpy, os
         assert 'get_include' in dir(numpy)
         path = numpy.get_include()
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -31,6 +31,12 @@
     result = list(cursor)
     assert result == [(42,)]
 
+def test_connect_takes_same_positional_args_as_Connection(con):
+    from inspect import getargspec
+    clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
+    conargs = getargspec(_sqlite3.connect).args
+    assert clsargs == conargs
+
 def test_total_changes_after_close(con):
     con.close()
     pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -26,6 +26,8 @@
 class Lock(W_Root):
     "A box around an interp-level lock object."
 
+    _immutable_fields_ = ["lock"]
+
     def __init__(self, space):
         self.space = space
         try:
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -424,21 +424,24 @@
     x = w_float1.floatval
     y = w_float2.floatval
 
+    return W_FloatObject(_pow(space, x, y))
+
+def _pow(space, x, y):
     # Sort out special cases here instead of relying on pow()
-    if y == 2.0:                      # special case for performance:
-        return W_FloatObject(x * x)   # x * x is always correct
+    if y == 2.0:       # special case for performance:
+        return x * x   # x * x is always correct
     if y == 0.0:
         # x**0 is 1, even 0**0
-        return W_FloatObject(1.0)
+        return 1.0
     if isnan(x):
         # nan**y = nan, unless y == 0
-        return W_FloatObject(x)
+        return x
     if isnan(y):
         # x**nan = nan, unless x == 1; x**nan = x
         if x == 1.0:
-            return W_FloatObject(1.0)
+            return 1.0
         else:
-            return W_FloatObject(y)
+            return y
     if isinf(y):
         # x**inf is: 0.0 if abs(x) < 1; 1.0 if abs(x) == 1; inf if
         # abs(x) > 1 (including case where x infinite)
@@ -447,11 +450,11 @@
         # abs(x) > 1 (including case where v infinite)
         x = abs(x)
         if x == 1.0:
-            return W_FloatObject(1.0)
+            return 1.0
         elif (y > 0.0) == (x > 1.0):
-            return W_FloatObject(INFINITY)
+            return INFINITY
         else:
-            return W_FloatObject(0.0)
+            return 0.0
     if isinf(x):
         # (+-inf)**w is: inf for w positive, 0 for w negative; in oth
         # cases, we need to add the appropriate sign if w is an odd
@@ -459,14 +462,14 @@
         y_is_odd = math.fmod(abs(y), 2.0) == 1.0
         if y > 0.0:
             if y_is_odd:
-                return W_FloatObject(x)
+                return x
             else:
-                return W_FloatObject(abs(x))
+                return abs(x)
         else:
             if y_is_odd:
-                return W_FloatObject(copysign(0.0, x))
+                return copysign(0.0, x)
             else:
-                return W_FloatObject(0.0)
+                return 0.0
 
     if x == 0.0:
         if y < 0.0:
@@ -480,7 +483,7 @@
     # -           pipermail/python-bugs-list/2003-March/016795.html
     if x < 0.0:
         if isnan(y):
-            return W_FloatObject(NAN)
+            return NAN
         if math.floor(y) != y:
             raise OperationError(space.w_ValueError,
                                  space.wrap("negative number cannot be "
@@ -494,9 +497,9 @@
     if x == 1.0:
         # (-1) ** large_integer also ends up here
         if negate_result:
-            return W_FloatObject(-1.0)
+            return -1.0
         else:
-            return W_FloatObject(1.0)
+            return 1.0
 
     try:
         # We delegate to our implementation of math.pow() the error detection.
@@ -510,7 +513,7 @@
 
     if negate_result:
         z = -z
-    return W_FloatObject(z)
+    return z
 
 
 def neg__Float(space, w_float1):
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -30,7 +30,7 @@
 from rpython.rlib import debug, jit, rerased
 from rpython.rlib.listsort import make_timsort_class
 from rpython.rlib.objectmodel import (
-    instantiate, newlist_hint, resizelist_hint, specialize)
+    instantiate, newlist_hint, resizelist_hint, specialize, import_from_mixin)
 from rpython.tool.sourcetools import func_with_new_name
 
 __all__ = ['W_ListObject', 'make_range_list', 'make_empty_list_with_size']
@@ -1170,7 +1170,6 @@
 
 
 class AbstractUnwrappedStrategy(object):
-    _mixin_ = True
 
     def wrap(self, unwrapped):
         raise NotImplementedError
@@ -1329,7 +1328,6 @@
 
     def setslice(self, w_list, start, step, slicelength, w_other):
         assert slicelength >= 0
-        items = self.unerase(w_list.lstorage)
 
         if self is self.space.fromcache(ObjectListStrategy):
             w_other = w_other._temporarily_as_objects()
@@ -1341,6 +1339,7 @@
             w_list.setslice(start, step, slicelength, w_other_as_object)
             return
 
+        items = self.unerase(w_list.lstorage)
         oldsize = len(items)
         len2 = w_other.length()
         if step == 1:  # Support list resizing for non-extended slices
@@ -1456,7 +1455,9 @@
         self.unerase(w_list.lstorage).reverse()
 
 
-class ObjectListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class ObjectListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "object"
 
@@ -1489,7 +1490,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class IntegerListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class IntegerListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0
     _applevel_repr = "int"
 
@@ -1520,7 +1523,30 @@
         return self.unerase(w_list.lstorage)
 
 
-class FloatListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+    _base_extend_from_list = _extend_from_list
+
+    def _extend_from_list(self, w_list, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            l = self.unerase(w_list.lstorage)
+            other = w_other.getitems_int()
+            assert other is not None
+            l += other
+            return
+        return self._base_extend_from_list(w_list, w_other)
+
+
+    _base_setslice = setslice
+
+    def setslice(self, w_list, start, step, slicelength, w_other):
+        if w_other.strategy is self.space.fromcache(RangeListStrategy):
+            storage = self.erase(w_other.getitems_int())
+            w_other = W_ListObject.from_storage_and_strategy(
+                    self.space, storage, self)
+        return self._base_setslice(w_list, start, step, slicelength, w_other)
+
+class FloatListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = 0.0
     _applevel_repr = "float"
 
@@ -1548,7 +1574,9 @@
             l.reverse()
 
 
-class StringListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class StringListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "str"
 
@@ -1579,7 +1607,9 @@
         return self.unerase(w_list.lstorage)
 
 
-class UnicodeListStrategy(AbstractUnwrappedStrategy, ListStrategy):
+class UnicodeListStrategy(ListStrategy):
+    import_from_mixin(AbstractUnwrappedStrategy)
+
     _none_value = None
     _applevel_repr = "unicode"
 
diff --git a/pypy/objspace/std/test/test_boolobject.py b/pypy/objspace/std/test/test_boolobject.py
--- a/pypy/objspace/std/test/test_boolobject.py
+++ b/pypy/objspace/std/test/test_boolobject.py
@@ -44,6 +44,13 @@
     def test_bool_int(self):
         assert int(True) is 1
         assert int(False) is 0
+        # XXX: broken
+        #assert True.__int__() is 1
+
+    def test_bool_long(self):
+        assert long(True) is 1L
+        assert long(False) is 0L
+        assert True.__long__() is 1L
 
     def test_bool_ops(self):
         assert True + True == 2
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -225,6 +225,15 @@
         l.setslice(0, 1, 2, W_ListObject(space, [w('a'), w(2), w(3)]))
         assert isinstance(l.strategy, ObjectListStrategy)
 
+    def test_setslice_int_range(self):
+        space = self.space
+        w = space.wrap
+        l = W_ListObject(space, [w(1), w(2), w(3)])
+        assert isinstance(l.strategy, IntegerListStrategy)
+        l.setslice(0, 1, 2, make_range_list(space, 5, 1, 4))
+        assert isinstance(l.strategy, IntegerListStrategy)
+
+
     def test_setslice_List(self):
         space = self.space
 
@@ -467,6 +476,12 @@
         l4 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3), self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         assert self.space.eq_w(l3, l4)
 
+    def test_add_of_range_and_int(self):
+        l1 = make_range_list(self.space, 0, 1, 100)
+        l2 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
+        l3 = self.space.add(l2, l1)
+        assert l3.strategy is l2.strategy
+
     def test_mul(self):
         l1 = W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)])
         l2 = l1.mul(2)
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -197,10 +197,10 @@
 
 W_AbstractTupleObject.typedef = StdTypeDef(
     "tuple",
-    __doc__ = '''tuple() -> an empty tuple
+    __doc__ = """tuple() -> an empty tuple
 tuple(sequence) -> tuple initialized from sequence's items
 
-If the argument is a tuple, the return value is the same object.''',
+If the argument is a tuple, the return value is the same object.""",
     __new__ = interp2app(W_AbstractTupleObject.descr_new),
     __repr__ = interp2app(W_AbstractTupleObject.descr_repr),
     __hash__ = interpindirect2app(W_AbstractTupleObject.descr_hash),
diff --git a/pypy/pytest-A.py b/pypy/pytest-A.py
--- a/pypy/pytest-A.py
+++ b/pypy/pytest-A.py
@@ -5,7 +5,6 @@
     'arm': ['interpreter/astcompiler/test',
             'interpreter/pyparser/test',
             'interpreter/test',
-            'interpreter/test2',
             'module/test_lib_pypy',
             'objspace/std/test',
     ],
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3442,6 +3442,29 @@
 
         a.build_types(f, [str])
 
+    def test_negative_number_find(self):
+        def f(s, e):
+            return "xyz".find("x", s, e)
+
+        a = self.RPythonAnnotator()
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        def f(s, e):
+            return "xyz".rfind("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+
+        def f(s, e):
+            return "xyz".count("x", s, e)
+
+        py.test.raises(annmodel.AnnotatorError, "a.build_types(f, [int, int])")
+        a.build_types(f, [annmodel.SomeInteger(nonneg=True),
+                          annmodel.SomeInteger(nonneg=True)])
+        
+
     def test_setslice(self):
         def f():
             lst = [2, 5, 7]
@@ -4080,7 +4103,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify instances with no common base class" 
+        assert ("RPython cannot unify instances with no common base class"
                 in exc.value.msg)
 
     def test_unionerror_iters(self):
@@ -4096,7 +4119,7 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot unify incompatible iterator variants" in 
+        assert ("RPython cannot unify incompatible iterator variants" in
                 exc.value.msg)
 
     def test_variable_getattr(self):
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -10,7 +10,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeFloat, SomeIterator,
     SomePBC, SomeTypedAddressAccess, SomeAddress, SomeType, s_ImpossibleValue,
     s_Bool, s_None, unionof, missing_operation, add_knowntypedata,
-    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString)
+    HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper
 from rpython.annotator import builtin
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
@@ -333,12 +333,13 @@
         check_negative_slice(s_start, s_stop)
         lst.listdef.resize()
 
-def check_negative_slice(s_start, s_stop):
+def check_negative_slice(s_start, s_stop, error="slicing"):
     if isinstance(s_start, SomeInteger) and not s_start.nonneg:
-        raise AnnotatorError("slicing: not proven to have non-negative start")
+        raise AnnotatorError("%s: not proven to have non-negative start" %
+                             error)
     if isinstance(s_stop, SomeInteger) and not s_stop.nonneg and \
            getattr(s_stop, 'const', 0) != -1:
-        raise AnnotatorError("slicing: not proven to have non-negative stop")
+        raise AnnotatorError("%s: not proven to have non-negative stop" % error)
 
 
 class __extend__(SomeDict):
@@ -448,12 +449,15 @@
         return s_Bool
 
     def method_find(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "find")
         return SomeInteger()
 
     def method_rfind(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "rfind")
         return SomeInteger()
 
     def method_count(str, frag, start=None, end=None):
+        check_negative_slice(start, end, "count")
         return SomeInteger(nonneg=True)
 
     def method_strip(str, chr):
@@ -520,6 +524,11 @@
     op_contains.can_only_throw = []
 
 
+class __extend__(SomeByteArray):
+    def getslice(ba, s_start, s_stop):
+        check_negative_slice(s_start, s_stop)
+        return SomeByteArray()
+
 class __extend__(SomeUnicodeString):
     def method_encode(uni, s_enc):
         if not s_enc.is_constant():
diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py
--- a/rpython/flowspace/model.py
+++ b/rpython/flowspace/model.py
@@ -252,6 +252,23 @@
         from rpython.translator.tool.graphpage import try_show
         try_show(self)
 
+    def get_graph(self):
+        import gc
+        pending = [self]   # pending blocks

From noreply at buildbot.pypy.org  Thu Oct 17 18:22:34 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu, 17 Oct 2013 18:22:34 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Fix typo
Message-ID: <20131017162234.897F41C30DC@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-array_prepare_-array_wrap
Changeset: r67463:4b9adb9cb0db
Date: 2013-10-17 18:21 +0200
http://bitbucket.org/pypy/pypy/changeset/4b9adb9cb0db/

Log:	Fix typo

diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -357,7 +357,7 @@
         elif name == 'logical_or':
             self.done_func = done_if_true
         else:
-            self.done_func = Non
+            self.done_func = None
 
     def are_common_types(self, dtype1, dtype2):
         if dtype1.is_complex_type() and dtype2.is_complex_type():

From noreply at buildbot.pypy.org  Thu Oct 17 23:02:01 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 23:02:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: make this error message match numpy
Message-ID: <20131017210201.B02731C08A6@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67464:b945ca0cf540
Date: 2013-10-17 04:28 -0400
http://bitbucket.org/pypy/pypy/changeset/b945ca0cf540/

Log:	make this error message match numpy

diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -117,12 +117,12 @@
                 shape[i] += axis_size
         a_dt = arr.get_dtype()
         if dtype.is_record_type() and a_dt.is_record_type():
-            #Record types must match
+            # Record types must match
             for f in dtype.fields:
                 if f not in a_dt.fields or \
                              dtype.fields[f] != a_dt.fields[f]:
                     raise OperationError(space.w_TypeError,
-                               space.wrap("record type mismatch"))
+                               space.wrap("invalid type promotion"))
         elif dtype.is_record_type() or a_dt.is_record_type():
             raise OperationError(space.w_TypeError,
                         space.wrap("invalid type promotion"))
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1639,7 +1639,7 @@
         exc = raises(TypeError, concatenate,
                             (zeros((2,), dtype=[('x', int), ('y', float)]),
                             (zeros((2,), dtype=[('x', float), ('y', float)]))))
-        assert str(exc.value).startswith('record type mismatch')
+        assert str(exc.value).startswith('invalid type promotion')
         exc = raises(TypeError, concatenate, ([1], zeros((2,),
                                             dtype=[('x', int), ('y', float)])))
         assert str(exc.value).startswith('invalid type promotion')

From noreply at buildbot.pypy.org  Thu Oct 17 23:02:07 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 23:02:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: clean up test_complex
Message-ID: <20131017210207.10F131C08A6@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67465:92fff1d93735
Date: 2013-10-17 06:00 -0400
http://bitbucket.org/pypy/pypy/changeset/92fff1d93735/

Log:	clean up test_complex

diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -10,7 +10,7 @@
 from rpython.rlib.rcomplex import c_pow
 
 
-def rAlmostEqual(a, b, rel_err=2e-15, abs_err=5e-323, msg='', isnumpy=False):
+def rAlmostEqual(a, b, rel_err=2e-15, abs_err=5e-323, msg=''):
     """Fail if the two floating-point numbers are not almost equal.
 
     Determine whether floating-point values a and b are equal to within
@@ -36,7 +36,7 @@
     # (in theory there are examples where it would be legitimate for a
     # and b to have opposite signs; in practice these hardly ever
     # occur).
-    if not a and not b and not isnumpy:
+    if not a and not b:
         # only check it if we are running on top of CPython >= 2.6
         if sys.version_info >= (2, 6) and copysign(1., a) != copysign(1., b):
             raise AssertionError( msg + \
@@ -112,8 +112,6 @@
                     (k, space.unwrap(v))
                     for k, v in kwargs.iteritems()
                 ])
-                if '__pypy__' not in sys.builtin_module_names:
-                    kwargs['isnumpy'] = True
                 return space.wrap(rAlmostEqual(*args, **kwargs))
             cls.w_rAlmostEqual = cls.space.wrap(interp2app(cls_rAlmostEqual))
             def cls_c_pow(space, args_w):
@@ -616,9 +614,9 @@
         import numpypy as np
         rAlmostEqual = self.rAlmostEqual
 
-        for complex_, abs_err, testcases in (\
-                 (np.complex128, 5e-323, self.testcases128),
-                 # (np.complex64,  5e-32,  self.testcases64),
+        for complex_, testcases in (
+                 (np.complex128, self.testcases128),
+                 #(np.complex64, self.testcases64),
                 ):
             for id, fn, ar, ai, er, ei, flags in testcases:
                 arg = complex_(complex(ar, ai))
@@ -647,7 +645,7 @@
                 if fn in ('log', 'log10'):
                     real_abs_err = 2e-15
                 else:
-                    real_abs_err = abs_err
+                    real_abs_err = 5e-323
 
                 error_message = (
                     '%s: %s(%r(%r, %r))\n'
@@ -660,9 +658,9 @@
                 # since rAlmostEqual is a wrapped function,
                 # convert arguments to avoid boxed values
                 rAlmostEqual(float(expected[0]), float(actual[0]),
-                               abs_err=real_abs_err, msg=error_message)
+                             abs_err=real_abs_err, msg=error_message)
                 rAlmostEqual(float(expected[1]), float(actual[1]),
-                                   msg=error_message)
+                             msg=error_message)
                 sys.stderr.write('.')
             sys.stderr.write('\n')
 

From noreply at buildbot.pypy.org  Thu Oct 17 23:02:08 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 23:02:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix sorting of unsigned arrays
Message-ID: <20131017210208.549A41C08A6@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67466:8596a76f1fd7
Date: 2013-10-17 16:59 -0400
http://bitbucket.org/pypy/pypy/changeset/8596a76f1fd7/

Log:	fix sorting of unsigned arrays

diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -8,7 +8,7 @@
 from rpython.rlib.rawstorage import raw_storage_getitem, raw_storage_setitem, \
         free_raw_storage, alloc_raw_storage
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import widen
 from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError
 from pypy.module.micronumpy.base import W_NDimArray
@@ -43,7 +43,7 @@
                                     + self.start + step * i)
                     v.append(_v)
             if comp_type == 'int':
-                v = intmask(v)
+                v = widen(v)
             elif comp_type == 'float':
                 v = float(v)
             elif comp_type == 'complex':
@@ -200,7 +200,7 @@
                                     + self.start + step * i)
                     v.append(_v)
             if comp_type == 'int':
-                v = intmask(v)
+                v = widen(v)
             elif comp_type == 'float':
                 v = float(v)
             elif comp_type == 'complex':
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -16,14 +16,17 @@
         assert array(2.0).argsort() == 0
         nnp = self.non_native_prefix
         for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
-                        nnp + 'i2', complex]:
+                      nnp + 'i2', complex]:
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            exp = list(a)
+            exp = sorted(range(len(exp)), key=exp.__getitem__)
             c = a.copy()
             res = a.argsort()
-            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
+            assert (res == exp).all(), \
                 'a,res,dtype %r,%r,%r' % (a,res,dtype)
             assert (a == c).all() # not modified
-            a = arange(100)
+
+            a = arange(100, dtype=dtype)
             assert (a.argsort() == a).all()
         raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
 
@@ -63,17 +66,18 @@
     def test_sort_dtypes(self):
         from numpypy import array, arange
         for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
-                        'i2', complex]:
+                      'i2', complex]:
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
-            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
+            b = sorted(list(a))
             c = a.copy()
             a.sort()
             assert (a == b).all(), \
                 'a,orig,dtype %r,%r,%r' % (a,c,dtype)
-        a = arange(100)
-        c = a.copy()
-        a.sort()
-        assert (a == c).all()
+
+            a = arange(100, dtype=dtype)
+            c = a.copy()
+            a.sort()
+            assert (a == c).all()
 
     def test_sort_dtypesi_nonnative(self):
         from numpypy import array

From noreply at buildbot.pypy.org  Thu Oct 17 23:12:33 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 17 Oct 2013 23:12:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix argsort for nan/inf
Message-ID: <20131017211233.A66AB1C0E26@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67467:31c7d69c6043
Date: 2013-10-17 17:11 -0400
http://bitbucket.org/pypy/pypy/changeset/31c7d69c6043/

Log:	fix argsort for nan/inf

diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -100,10 +100,15 @@
     if count < 2:
         def arg_lt(a, b):
             # Does numpy do <= ?
-            return a[0] < b[0]
+            return a[0] < b[0] or b[0] != b[0] and a[0] == a[0]
     else:
         def arg_lt(a, b):
             for i in range(count):
+                if b[0][i] != b[0][i] and a[0][i] == a[0][i]:
+                    return True
+                elif b[0][i] == b[0][i] and a[0][i] != a[0][i]:
+                    return False
+            for i in range(count):
                 if a[0][i] < b[0][i]:
                     return True
                 elif a[0][i] > b[0][i]:
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -108,6 +108,9 @@
         assert [isnan(bb) for bb in b] == [isnan(aa) for aa in a[::-1]]
         assert (b[:2] == a[::-1][:2]).all()
 
+        b = a.argsort()
+        assert (b == [2, 1, 0]).all()
+
         # check complex
         a = zeros(9, dtype=complex128)
         a.real += [nan, nan, nan, 1, 0, 1, 1, 0, 0]
@@ -117,6 +120,9 @@
         assert [isnan(bb) for bb in b] == [isnan(aa) for aa in a[::-1]]
         assert (b[:4] == a[::-1][:4]).all()
 
+        b = a.argsort()
+        assert (b == [8, 7, 6, 5, 4, 3, 2, 1, 0]).all()
+
         # all c scalar sorts use the same code with different types
         # so it suffices to run a quick check with one type. The number
         # of sorted items must be greater than ~50 to check the actual

From noreply at buildbot.pypy.org  Fri Oct 18 00:43:14 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 18 Oct 2013 00:43:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: some cleanups
Message-ID: <20131017224314.E92C61C01B0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67468:caf88c5b381f
Date: 2013-10-17 17:34 -0400
http://bitbucket.org/pypy/pypy/changeset/caf88c5b381f/

Log:	some cleanups

diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -323,7 +323,8 @@
 
 all_types = (types.all_float_types + types.all_complex_types +
              types.all_int_types)
-all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
+all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__ and
+                                     not issubclass(i[0], types.BaseFloat16)]
 all_types = unrolling_iterable(all_types)
 
 class ArgSortCache(object):
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -1,16 +1,6 @@
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-class AppTestSupport(BaseNumpyAppTest):
-    def setup_class(cls):
-        import struct
-        BaseNumpyAppTest.setup_class.im_func(cls)
-        cls.w_data = cls.space.wrap(struct.pack('dddd', 1, 2, 3, 4))
-        cls.w_fdata = cls.space.wrap(struct.pack('f', 2.3))
-        cls.w_float16val = cls.space.wrap('\x00E') # 5.0 in float16
-        cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
-        cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
-        cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
-
+class AppTestSorting(BaseNumpyAppTest):
     def test_argsort_dtypes(self):
         from numpypy import array, arange
         assert array(2.0).argsort() == 0
@@ -30,7 +20,7 @@
             assert (a.argsort() == a).all()
         raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
 
-    def test_argsort_nd(self):
+    def test_argsort_ndim(self):
         from numpypy import array
         a = array([[4, 2], [1, 3]])
         assert (a.argsort() == [[1, 0], [0, 1]]).all()
@@ -79,7 +69,7 @@
             a.sort()
             assert (a == c).all()
 
-    def test_sort_dtypesi_nonnative(self):
+    def test_sort_nonnative(self):
         from numpypy import array
         nnp = self.non_native_prefix
         for dtype in [ nnp + 'i2']:
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -68,7 +68,6 @@
         )
     return dispatcher
 
-
 def raw_unary_op(func):
     specialize.argtype(1)(func)
     @functools.wraps(func)
@@ -1038,6 +1037,53 @@
         swapped_value = byteswap(rffi.cast(self.T, value))
         raw_storage_setitem(storage, i + offset, swapped_value)
 
+class BaseFloat16(Float):
+    _mixin_ = True
+
+    _attrs_ = ()
+    _STORAGE_T = rffi.USHORT
+    T = rffi.SHORT
+
+    BoxType = interp_boxes.W_Float16Box
+
+    @specialize.argtype(1)
+    def box(self, value):
+        return self.BoxType(rffi.cast(rffi.DOUBLE, value))
+
+    def runpack_str(self, s):
+        assert len(s) == 2
+        fval = unpack_float(s, native_is_bigendian)
+        return self.box(fval)
+
+    def default_fromstring(self, space):
+        return self.box(-1.0)
+
+    def byteswap(self, w_v):
+        value = self.unbox(w_v)
+        hbits = float_pack(value,2)
+        swapped = byteswap(rffi.cast(self._STORAGE_T, hbits))
+        return self.box(float_unpack(r_ulonglong(swapped), 2))
+
+class Float16(BaseType, BaseFloat16):
+    def _read(self, storage, i, offset):
+        hbits = raw_storage_getitem(self._STORAGE_T, storage, i + offset)
+        return float_unpack(r_ulonglong(hbits), 2)
+
+    def _write(self, storage, i, offset, value):
+        hbits = float_pack(value,2)
+        raw_storage_setitem(storage, i + offset,
+                rffi.cast(self._STORAGE_T, hbits))
+
+class NonNativeFloat16(BaseType, BaseFloat16):
+    def _read(self, storage, i, offset):
+        hbits = raw_storage_getitem(self._STORAGE_T, storage, i + offset)
+        return float_unpack(r_ulonglong(byteswap(hbits)), 2)
+
+    def _write(self, storage, i, offset, value):
+        hbits = float_pack(value,2)
+        raw_storage_setitem(storage, i + offset,
+                byteswap(rffi.cast(self._STORAGE_T, hbits)))
+
 class Float32(BaseType, Float):
     _attrs_ = ()
 
@@ -1169,22 +1215,22 @@
         real, imag = box.real, box.imag
         return real, imag
 
-    def store(self, arr, i, offset, box):
-        real, imag = self.unbox(box)
-        raw_storage_setitem(arr.storage, i+offset, real)
-        raw_storage_setitem(arr.storage,
-                i+offset+rffi.sizeof(self.T), imag)
-
     def _read(self, storage, i, offset):
         real = raw_storage_getitem(self.T, storage, i + offset)
-        imag = raw_storage_getitem(self.T, storage,
-                              i + offset + rffi.sizeof(self.T))
+        imag = raw_storage_getitem(self.T, storage, i + offset + rffi.sizeof(self.T))
         return real, imag
 
     def read(self, arr, i, offset, dtype=None):
         real, imag = self._read(arr.storage, i, offset)
         return self.box_complex(real, imag)
 
+    def _write(self, storage, i, offset, value):
+        raw_storage_setitem(storage, i + offset, value[0])
+        raw_storage_setitem(storage, i + offset + rffi.sizeof(self.T), value[1])
+
+    def store(self, arr, i, offset, box):
+        self._write(arr.storage, i, offset, self.unbox(box))
+
     @complex_binary_op
     def add(self, v1, v2):
         return rcomplex.c_add(v1, v2)
@@ -1621,7 +1667,6 @@
     def zeros_like(self, v):
         return 0, 0
 
-
 class Complex64(ComplexFloating, BaseType):
     _attrs_ = ()
 
@@ -1675,7 +1720,6 @@
 
     NonNativeComplexLong = ComplexLong
 
-
 class BaseStringType(object):
     _mixin_ = True
 
@@ -1883,7 +1927,6 @@
 NonNativeVoidType = VoidType
 
 class RecordType(BaseType):
-
     T = lltype.Char
 
     def __init__(self, offsets_and_fields, size):
@@ -1967,50 +2010,3 @@
                 all_complex_types.append((tp, 'complex'))
 _setup()
 del _setup
-
-class BaseFloat16(Float):
-    _mixin_ = True
-
-    _attrs_ = ()
-    _STORAGE_T = rffi.USHORT
-    T = rffi.SHORT
-
-    BoxType = interp_boxes.W_Float16Box
-
-    @specialize.argtype(1)
-    def box(self, value):
-        return self.BoxType(rffi.cast(rffi.DOUBLE, value))
-
-    def runpack_str(self, s):
-        assert len(s) == 2
-        fval = unpack_float(s, native_is_bigendian)
-        return self.box(fval)
-
-    def default_fromstring(self, space):
-        return self.box(-1.0)
-
-    def byteswap(self, w_v):
-        value = self.unbox(w_v)
-        hbits = float_pack(value,2)
-        swapped = byteswap(rffi.cast(self._STORAGE_T, hbits))
-        return self.box(float_unpack(r_ulonglong(swapped), 2))
-
-class Float16(BaseType, BaseFloat16):
-    def _read(self, storage, i, offset):
-        hbits = raw_storage_getitem(self._STORAGE_T, storage, i + offset)
-        return float_unpack(r_ulonglong(hbits), 2)
-
-    def _write(self, storage, i, offset, value):
-        hbits = float_pack(value,2)
-        raw_storage_setitem(storage, i + offset,
-                rffi.cast(self._STORAGE_T, hbits))
-
-class NonNativeFloat16(BaseType, BaseFloat16):
-    def _read(self, storage, i, offset):
-        hbits = raw_storage_getitem(self._STORAGE_T, storage, i + offset)
-        return float_unpack(r_ulonglong(byteswap(hbits)), 2)
-
-    def _write(self, storage, i, offset, value):
-        hbits = float_pack(value,2)
-        raw_storage_setitem(storage, i + offset,
-                byteswap(rffi.cast(self._STORAGE_T, hbits)))

From noreply at buildbot.pypy.org  Fri Oct 18 01:19:51 2013
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 18 Oct 2013 01:19:51 +0200 (CEST)
Subject: [pypy-commit] pypy cpyext-int: Fix tests around PyIntObject:
Message-ID: <20131017231951.0F43C1D2309@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: cpyext-int
Changeset: r67469:152c8a84b76d
Date: 2013-10-18 01:19 +0200
http://bitbucket.org/pypy/pypy/changeset/152c8a84b76d/

Log:	Fix tests around PyIntObject:
	- Revert the changes made to signature, to match CPython again.
	- Fix the test
	- Really fill the PyIntObject::ob_ival member (with an "attach"
	descriptor)

diff --git a/pypy/module/cpyext/intobject.py b/pypy/module/cpyext/intobject.py
--- a/pypy/module/cpyext/intobject.py
+++ b/pypy/module/cpyext/intobject.py
@@ -21,8 +21,17 @@
     "Type description of PyIntObject"
     make_typedescr(space.w_int.instancetypedef,
                    basestruct=PyIntObject.TO,
+                   attach=int_attach,
                    realize=int_realize)
 
+def int_attach(space, py_obj, w_obj):
+    """
+    Fills a newly allocated PyIntObject with the given int object. The
+    value must not be modified.
+    """
+    py_int = rffi.cast(PyIntObject, py_obj)
+    py_int.c_ob_ival = space.int_w(w_obj)
+
 def int_realize(space, obj):
     intval = rffi.cast(lltype.Signed, rffi.cast(PyIntObject, obj).c_ob_ival)
     w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
@@ -41,10 +50,10 @@
     as defined in the system header files)."""
     return sys.maxint
 
- at cpython_api([lltype.Signed], PyIntObject)
+ at cpython_api([lltype.Signed], PyObject)
 def PyInt_FromLong(space, ival):
     """Create a new integer object with a value of ival.
-
+    
     """
     return space.wrap(ival)
 
@@ -117,7 +126,7 @@
 
 LONG_MAX = int(LONG_TEST - 1)
 
- at cpython_api([rffi.SIZE_T], PyIntObject)
+ at cpython_api([rffi.SIZE_T], PyObject)
 def PyInt_FromSize_t(space, ival):
     """Create a new integer object with a value of ival. If the value exceeds
     LONG_MAX, a long integer object is returned.
@@ -126,7 +135,7 @@
         return space.wrap(intmask(ival))
     return space.wrap(ival)
 
- at cpython_api([Py_ssize_t], PyIntObject)
+ at cpython_api([Py_ssize_t], PyObject)
 def PyInt_FromSsize_t(space, ival):
     """Create a new integer object with a value of ival. If the value is larger
     than LONG_MAX or smaller than LONG_MIN, a long integer object is
@@ -134,7 +143,7 @@
     """
     return space.wrap(ival)
 
- at cpython_api([CONST_STRING, rffi.CCHARPP, rffi.INT_real], PyIntObject)
+ at cpython_api([CONST_STRING, rffi.CCHARPP, rffi.INT_real], PyObject)
 def PyInt_FromString(space, str, pend, base):
     """Return a new PyIntObject or PyLongObject based on the string
     value in str, which is interpreted according to the radix in base.  If
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -292,10 +292,13 @@
                 #prove it works for ints
                 ("test_int", "METH_NOARGS",
                 """
-                PyIntObject * obj = PyInt_FromLong(42);
-                if ( PyInt_Check(obj))
-                    return obj;
-                PyObject * val = PyInt_FromLong(obj->ob_ival);
+                PyObject * obj = PyInt_FromLong(42);
+                if (!PyInt_Check(obj)) {
+                    Py_DECREF(obj);
+                    PyErr_SetNone(PyExc_ValueError);
+                    return NULL;
+                }
+                PyObject * val = PyInt_FromLong(((PyIntObject *)obj)->ob_ival);
                 Py_DECREF(obj);
                 return val;
                 """

From noreply at buildbot.pypy.org  Fri Oct 18 01:59:48 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 18 Oct 2013 01:59:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix reciprocal on 32bit
Message-ID: <20131017235948.1AC581C01B0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67470:90289b2c633b
Date: 2013-10-17 19:59 -0400
http://bitbucket.org/pypy/pypy/changeset/90289b2c633b/

Log:	fix reciprocal on 32bit

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -315,16 +315,16 @@
         for i in range(4):
             assert b[i] == reference[i]
 
-        for dtype in ['int8', 'int16', 'int32', 'int64',
-                      'uint8', 'uint16', 'uint32', 'uint64']:
+        for dtype in 'bBhHiIlLqQ':
+            a = array([-2, -1, 0, 1, 2], dtype)
             reference = [0, -1, 0, 1, 0]
+            dtype = a.dtype.name
             if dtype[0] == 'u':
                 reference[1] = 0
             elif dtype == 'int32':
                     reference[2] = -2147483648
             elif dtype == 'int64':
                     reference[2] = -9223372036854775808
-            a = array([-2, -1, 0, 1, 2], dtype)
             b = reciprocal(a)
             assert (b == reference).all()
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -510,7 +510,7 @@
         ans = 0
         if raw == 0:
             # XXX good place to warn
-            if self.T is rffi.INT or self.T is rffi.LONG:
+            if self.T is rffi.INT or self.T is rffi.LONG or self.T is rffi.LONGLONG:
                 ans = most_neg_value_of(self.T)
         elif abs(raw) == 1:
             ans = raw

From noreply at buildbot.pypy.org  Fri Oct 18 03:04:45 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 18 Oct 2013 03:04:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: these tests work now, enable
Message-ID: <20131018010445.9D28E1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67471:356aaba5c3ce
Date: 2013-10-17 21:04 -0400
http://bitbucket.org/pypy/pypy/changeset/356aaba5c3ce/

Log:	these tests work now, enable

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1295,9 +1295,7 @@
         assert a.max() == 5.7
         b = array([])
         raises(ValueError, "b.max()")
-
-        if 0: # XXX too pedantic
-            assert list(zeros((0, 2)).max(axis=1)) == []
+        assert list(zeros((0, 2)).max(axis=1)) == []
 
     def test_max_add(self):
         from numpypy import array
@@ -1310,9 +1308,7 @@
         assert a.min() == -3.0
         b = array([])
         raises(ValueError, "b.min()")
-
-        if 0: # XXX too pedantic
-            assert list(zeros((0, 2)).min(axis=1)) == []
+        assert list(zeros((0, 2)).min(axis=1)) == []
 
     def test_argmax(self):
         from numpypy import array

From noreply at buildbot.pypy.org  Fri Oct 18 06:05:59 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 18 Oct 2013 06:05:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix dtype descr_eq
Message-ID: <20131018040559.E004E1C08A6@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67472:05a9b19c2e61
Date: 2013-10-17 21:33 -0400
http://bitbucket.org/pypy/pypy/changeset/05a9b19c2e61/

Log:	fix dtype descr_eq

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -148,7 +148,11 @@
 
     def eq(self, space, w_other):
         w_other = space.call_function(space.gettypefor(W_Dtype), w_other)
-        return space.is_w(self, w_other)
+        if space.is_w(self, w_other):
+            return True
+        if isinstance(w_other, W_Dtype):
+            return space.eq_w(self.descr_reduce(space), w_other.descr_reduce(space))
+        return False
 
     def descr_eq(self, space, w_other):
         return space.wrap(self.eq(space, w_other))
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -703,7 +703,7 @@
         from numpypy import dtype
         nnp = self.non_native_prefix
         byteorder = self.native_prefix
-        assert dtype('i8') == dtype(byteorder + 'i8') == dtype('=i8') # XXX should be equal == dtype(long)
+        assert dtype('i8') == dtype(byteorder + 'i8') == dtype('=i8') == dtype(long)
         assert dtype(nnp + 'i8') != dtype('i8')
         assert dtype(nnp + 'i8').byteorder == nnp
         assert dtype('=i8').byteorder == '='

From noreply at buildbot.pypy.org  Fri Oct 18 06:06:01 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 18 Oct 2013 06:06:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: clean up some stuff in micronumpy
Message-ID: <20131018040601.2353C1C0E26@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67473:0ab4a0d7455f
Date: 2013-10-18 00:04 -0400
http://bitbucket.org/pypy/pypy/changeset/0ab4a0d7455f/

Log:	clean up some stuff in micronumpy

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -63,6 +63,7 @@
 
 class PrimitiveBox(Box):
     _mixin_ = True
+    _immutable_fields_ = ['value']
 
     def __init__(self, value):
         self.value = value
@@ -82,11 +83,11 @@
         ret = builder.build()
 
         lltype.free(value, flavor="raw")
-
         return ret
 
 class ComplexBox(Box):
     _mixin_ = True
+    _immutable_fields_ = ['real', 'imag']
 
     def __init__(self, real, imag=0.):
         self.real = real
@@ -111,12 +112,10 @@
         ret = builder.build()
 
         lltype.free(value, flavor="raw")
-
         return ret
 
+
 class W_GenericBox(W_Root):
-    _attrs_ = ()
-
     def descr__new__(space, w_subtype, __args__):
         raise operationerrfmt(space.w_TypeError,
                               "cannot create '%N' instances",
@@ -125,12 +124,21 @@
     def get_dtype(self, space):
         return self._get_dtype(space)
 
+    def item(self, space):
+        return self.get_dtype(space).itemtype.to_builtin_type(space, self)
+
     def descr_str(self, space):
         return space.wrap(self.get_dtype(space).itemtype.str_format(self))
 
     def descr_format(self, space, w_spec):
         return space.format(self.item(space), w_spec)
 
+    def descr_hash(self, space):
+        return space.hash(self.item(space))
+
+    def descr_index(self, space):
+        return space.index(self.item(space))
+
     def descr_int(self, space):
         box = self.convert_to(W_LongBox._get_dtype(space))
         assert isinstance(box, W_LongBox)
@@ -222,12 +230,6 @@
         w_remainder = self.descr_rmod(space, w_other)
         return space.newtuple([w_quotient, w_remainder])
 
-    def descr_hash(self, space):
-        return space.hash(self.item(space))
-
-    def item(self, space):
-        return self.get_dtype(space).itemtype.to_builtin_type(space, self)
-
     def descr_any(self, space):
         value = space.is_true(self)
         return self.get_dtype(space).box(value)
@@ -260,7 +262,7 @@
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
 class W_NumberBox(W_GenericBox):
-    _attrs_ = ()
+    pass
 
 class W_IntegerBox(W_NumberBox):
     def int_w(self, space):
@@ -309,10 +311,10 @@
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter('ulonglong')
 
 class W_InexactBox(W_NumberBox):
-    _attrs_ = ()
+    pass
 
 class W_FloatingBox(W_InexactBox):
-    _attrs_ = ()
+    pass
 
 class W_Float16Box(W_FloatingBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float16")
@@ -323,9 +325,42 @@
 class W_Float64Box(W_FloatingBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float64")
 
+class W_ComplexFloatingBox(W_InexactBox):
+    def descr_get_real(self, space):
+        dtype = self._COMPONENTS_BOX._get_dtype(space)
+        box = self.convert_real_to(dtype)
+        assert isinstance(box, self._COMPONENTS_BOX)
+        return space.wrap(box)
+
+    def descr_get_imag(self, space):
+        dtype = self._COMPONENTS_BOX._get_dtype(space)
+        box = self.convert_imag_to(dtype)
+        assert isinstance(box, self._COMPONENTS_BOX)
+        return space.wrap(box)
+
+class W_Complex64Box(ComplexBox, W_ComplexFloatingBox):
+    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex64")
+    _COMPONENTS_BOX = W_Float32Box
+
+class W_Complex128Box(ComplexBox, W_ComplexFloatingBox):
+    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex128")
+    _COMPONENTS_BOX = W_Float64Box
+
+if long_double_size == 8:
+    W_FloatLongBox = W_Float64Box
+    W_ComplexLongBox = W_Complex128Box
+
+elif long_double_size in (12, 16):
+    class W_FloatLongBox(W_FloatingBox, PrimitiveBox):
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % (long_double_size * 8))
+
+    class W_ComplexLongBox(ComplexBox, W_ComplexFloatingBox):
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % (long_double_size * 16))
+        _COMPONENTS_BOX = W_FloatLongBox
+
 class W_FlexibleBox(W_GenericBox):
-    _attrs_ = ['ofs', 'dtype', 'arr']
-    _immutable_fields_ = ['ofs']
+    _immutable_fields_ = ['arr', 'ofs', 'dtype']
+
     def __init__(self, arr, ofs, dtype):
         self.arr = arr # we have to keep array alive
         self.ofs = ofs
@@ -334,11 +369,6 @@
     def get_dtype(self, space):
         return self.arr.dtype
 
- at unwrap_spec(self=W_GenericBox)
-def descr_index(space, self):
-    return space.index(self.item(space))
-
-
 class W_VoidBox(W_FlexibleBox):
     def descr_getitem(self, space, w_item):
         from pypy.module.micronumpy.types import VoidType
@@ -388,7 +418,6 @@
         # XXX assert dtype is str type
         return self
 
-
 class W_StringBox(W_CharacterBox):
     def descr__new__string_box(space, w_subtype, w_arg):
         from pypy.module.micronumpy.interp_dtype import new_string_dtype
@@ -398,7 +427,6 @@
             arr.storage[i] = arg[i]
         return W_StringBox(arr, 0, arr.dtype)
 
-
 class W_UnicodeBox(W_CharacterBox):
     def descr__new__unicode_box(space, w_subtype, w_arg):
         raise OperationError(space.w_NotImplementedError, space.wrap("Unicode is not supported yet"))
@@ -413,45 +441,6 @@
         #    arr.storage[i] = arg[i]
         return W_UnicodeBox(arr, 0, arr.dtype)
 
-
-class W_ComplexFloatingBox(W_InexactBox):
-    _attrs_ = ()
-    def descr_get_real(self, space):
-        dtype = self._COMPONENTS_BOX._get_dtype(space)
-        box = self.convert_real_to(dtype)
-        assert isinstance(box, self._COMPONENTS_BOX)
-        return space.wrap(box)
-
-    def descr_get_imag(self, space):
-        dtype = self._COMPONENTS_BOX._get_dtype(space)
-        box = self.convert_imag_to(dtype)
-        assert isinstance(box, self._COMPONENTS_BOX)
-        return space.wrap(box)
-
-
-class W_Complex64Box(ComplexBox, W_ComplexFloatingBox):
-    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex64")
-    _COMPONENTS_BOX = W_Float32Box
-
-
-class W_Complex128Box(ComplexBox, W_ComplexFloatingBox):
-    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex128")
-    _COMPONENTS_BOX = W_Float64Box
-
-
-if long_double_size == 8:
-    W_FloatLongBox = W_Float64Box
-    W_ComplexLongBox = W_Complex128Box
-
-elif long_double_size in (12, 16):
-    class W_FloatLongBox(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % (long_double_size * 8))
-
-    class W_ComplexLongBox(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % (long_double_size * 16))
-        _COMPONENTS_BOX = W_FloatLongBox
-
-
 W_GenericBox.typedef = TypeDef("generic",
     __module__ = "numpypy",
 
@@ -521,7 +510,7 @@
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_BoolBox.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_BoolBox.descr_index),
     __reduce__ = interp2app(W_BoolBox.descr_reduce),
 )
 
@@ -544,49 +533,49 @@
 W_Int8Box.typedef = TypeDef("int8", W_SignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int8Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int8Box.descr_index),
     __reduce__ = interp2app(W_Int8Box.descr_reduce),
 )
 
 W_UInt8Box.typedef = TypeDef("uint8", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt8Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt8Box.descr_index),
     __reduce__ = interp2app(W_UInt8Box.descr_reduce),
 )
 
 W_Int16Box.typedef = TypeDef("int16", W_SignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int16Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int16Box.descr_index),
     __reduce__ = interp2app(W_Int16Box.descr_reduce),
 )
 
 W_UInt16Box.typedef = TypeDef("uint16", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt16Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt16Box.descr_index),
     __reduce__ = interp2app(W_UInt16Box.descr_reduce),
 )
 
 W_Int32Box.typedef = TypeDef("int32", (W_SignedIntegerBox.typedef,) + MIXIN_32,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int32Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int32Box.descr_index),
     __reduce__ = interp2app(W_Int32Box.descr_reduce),
 )
 
 W_UInt32Box.typedef = TypeDef("uint32", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt32Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt32Box.descr_index),
     __reduce__ = interp2app(W_UInt32Box.descr_reduce),
 )
 
 W_Int64Box.typedef = TypeDef("int64", (W_SignedIntegerBox.typedef,) + MIXIN_64,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int64Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int64Box.descr_index),
     __reduce__ = interp2app(W_Int64Box.descr_reduce),
 )
 
@@ -600,7 +589,7 @@
 W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt64Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt64Box.descr_index),
     __reduce__ = interp2app(W_UInt64Box.descr_reduce),
 )
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -18,10 +18,8 @@
 
 
 class W_Ufunc(W_Root):
-    _attrs_ = ["name", "promote_to_float", "promote_bools", "identity",
-               "allow_bool", "allow_complex", "complex_to_float"]
-    _immutable_fields_ = ["promote_to_float", "promote_bools", "name",
-            "allow_bool", "allow_complex", "complex_to_float"]
+    _immutable_fields_ = ["name", "promote_to_float", "promote_bools", "identity",
+            "int_only", "allow_bool", "allow_complex", "complex_to_float"]
 
     def __init__(self, name, promote_to_float, promote_bools, identity,
                  int_only, allow_bool, allow_complex, complex_to_float):
@@ -254,14 +252,12 @@
         return res
 
 class W_Ufunc1(W_Ufunc):
+    _immutable_fields_ = ["func", "bool_result"]
     argcount = 1
 
-    _immutable_fields_ = ["func", "name"]
-
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None, bool_result=False, int_only=False,
-        allow_bool=True, allow_complex=True, complex_to_float=False):
-
+            identity=None, bool_result=False, int_only=False,
+            allow_bool=True, allow_complex=True, complex_to_float=False):
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity,
                          int_only, allow_bool, allow_complex, complex_to_float)
         self.func = func
@@ -322,13 +318,12 @@
 
 
 class W_Ufunc2(W_Ufunc):
-    _immutable_fields_ = ["comparison_func", "func", "name", "int_only"]
+    _immutable_fields_ = ["func", "comparison_func", "done_func"]
     argcount = 2
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None, comparison_func=False, int_only=False,
-        allow_bool=True, allow_complex=True, complex_to_float=False):
-
+            identity=None, comparison_func=False, int_only=False,
+            allow_bool=True, allow_complex=True, complex_to_float=False):
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity,
                          int_only, allow_bool, allow_complex, complex_to_float)
         self.func = func
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -115,8 +115,6 @@
     return dispatcher
 
 class BaseType(object):
-    _attrs_ = ()
-
     SortRepr = None # placeholders for sorting classes, overloaded in sort.py
     Sort = None
 
@@ -323,8 +321,6 @@
         raw_storage_setitem(storage, i + offset, value)
 
 class Bool(BaseType, Primitive):
-    _attrs_ = ()
-
     T = lltype.Bool
     BoxType = interp_boxes.W_BoolBox
     format_code = "?"
@@ -541,101 +537,75 @@
     _mixin_ = True
 
 class Int8(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.SIGNEDCHAR
     BoxType = interp_boxes.W_Int8Box
     format_code = "b"
+
 NonNativeInt8 = Int8
 
 class UInt8(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.UCHAR
     BoxType = interp_boxes.W_UInt8Box
     format_code = "B"
+
 NonNativeUInt8 = UInt8
 
 class Int16(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.SHORT
     BoxType = interp_boxes.W_Int16Box
     format_code = "h"
 
 class NonNativeInt16(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.SHORT
     BoxType = interp_boxes.W_Int16Box
     format_code = "h"
 
 class UInt16(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.USHORT
     BoxType = interp_boxes.W_UInt16Box
     format_code = "H"
 
 class NonNativeUInt16(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.USHORT
     BoxType = interp_boxes.W_UInt16Box
     format_code = "H"
 
 class Int32(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.INT
     BoxType = interp_boxes.W_Int32Box
     format_code = "i"
 
 class NonNativeInt32(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.INT
     BoxType = interp_boxes.W_Int32Box
     format_code = "i"
 
 class UInt32(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.UINT
     BoxType = interp_boxes.W_UInt32Box
     format_code = "I"
 
 class NonNativeUInt32(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.UINT
     BoxType = interp_boxes.W_UInt32Box
     format_code = "I"
 
 class Long(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.LONG
     BoxType = interp_boxes.W_LongBox
     format_code = "l"
 
 class NonNativeLong(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.LONG
     BoxType = interp_boxes.W_LongBox
     format_code = "l"
 
 class ULong(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.ULONG
     BoxType = interp_boxes.W_ULongBox
     format_code = "L"
 
 class NonNativeULong(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.ULONG
     BoxType = interp_boxes.W_ULongBox
     format_code = "L"
@@ -654,8 +624,6 @@
     return self.box(value)
 
 class Int64(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.LONGLONG
     BoxType = interp_boxes.W_Int64Box
     format_code = "q"
@@ -663,8 +631,6 @@
     _coerce = func_with_new_name(_int64_coerce, '_coerce')
 
 class NonNativeInt64(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.LONGLONG
     BoxType = interp_boxes.W_Int64Box
     format_code = "q"
@@ -685,8 +651,6 @@
     return self.box(value)
 
 class UInt64(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.ULONGLONG
     BoxType = interp_boxes.W_UInt64Box
     format_code = "Q"
@@ -694,8 +658,6 @@
     _coerce = func_with_new_name(_uint64_coerce, '_coerce')
 
 class NonNativeUInt64(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.ULONGLONG
     BoxType = interp_boxes.W_UInt64Box
     format_code = "Q"
@@ -1040,10 +1002,8 @@
 class BaseFloat16(Float):
     _mixin_ = True
 
-    _attrs_ = ()
     _STORAGE_T = rffi.USHORT
     T = rffi.SHORT
-
     BoxType = interp_boxes.W_Float16Box
 
     @specialize.argtype(1)
@@ -1085,15 +1045,11 @@
                 byteswap(rffi.cast(self._STORAGE_T, hbits)))
 
 class Float32(BaseType, Float):
-    _attrs_ = ()
-
     T = rffi.FLOAT
     BoxType = interp_boxes.W_Float32Box
     format_code = "f"
 
 class NonNativeFloat32(BaseType, NonNativeFloat):
-    _attrs_ = ()
-
     T = rffi.FLOAT
     BoxType = interp_boxes.W_Float32Box
     format_code = "f"
@@ -1107,22 +1063,17 @@
         return bool(v)
 
 class Float64(BaseType, Float):
-    _attrs_ = ()
-
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
     format_code = "d"
 
 class NonNativeFloat64(BaseType, NonNativeFloat):
-    _attrs_ = ()
-
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
     format_code = "d"
 
 class ComplexFloating(object):
     _mixin_ = True
-    _attrs_ = ()
 
     def _coerce(self, space, w_item):
         w_item = space.call_function(space.w_complex, w_item)
@@ -1668,8 +1619,6 @@
         return 0, 0
 
 class Complex64(ComplexFloating, BaseType):
-    _attrs_ = ()
-
     T = rffi.FLOAT
     BoxType = interp_boxes.W_Complex64Box
     ComponentBoxType = interp_boxes.W_Float32Box
@@ -1677,8 +1626,6 @@
 NonNativeComplex64 = Complex64
 
 class Complex128(ComplexFloating, BaseType):
-    _attrs_ = ()
-
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Complex128Box
     ComponentBoxType = interp_boxes.W_Float64Box
@@ -1688,13 +1635,12 @@
 if interp_boxes.long_double_size == 8:
     FloatLong = Float64
     NonNativeFloatLong = NonNativeFloat64
+
     ComplexLong = Complex128
     NonNativeComplexLong = NonNativeComplex128
 
 elif interp_boxes.long_double_size in (12, 16):
     class FloatLong(BaseType, Float):
-        _attrs_ = ()
-
         T = rffi.LONGDOUBLE
         BoxType = interp_boxes.W_FloatLongBox
 
@@ -1712,8 +1658,6 @@
     NonNativeFloatLong = FloatLong
 
     class ComplexLong(ComplexFloating, BaseType):
-        _attrs_ = ()
-
         T = rffi.LONGDOUBLE
         BoxType = interp_boxes.W_ComplexLongBox
         ComponentBoxType = interp_boxes.W_FloatLongBox

From noreply at buildbot.pypy.org  Fri Oct 18 07:03:18 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 18 Oct 2013 07:03:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: more cleanups for numpypy
Message-ID: <20131018050318.831331C01F5@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67474:0032c1800df8
Date: 2013-10-18 00:39 -0400
http://bitbucket.org/pypy/pypy/changeset/0032c1800df8/

Log:	more cleanups for numpypy

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -116,6 +116,8 @@
 
 
 class W_GenericBox(W_Root):
+    _attrs_ = []
+
     def descr__new__(space, w_subtype, __args__):
         raise operationerrfmt(space.w_TypeError,
                               "cannot create '%N' instances",
@@ -359,6 +361,7 @@
         _COMPONENTS_BOX = W_FloatLongBox
 
 class W_FlexibleBox(W_GenericBox):
+    _attrs_ = ['arr', 'ofs', 'dtype']
     _immutable_fields_ = ['arr', 'ofs', 'dtype']
 
     def __init__(self, arr, ofs, dtype):
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -1,4 +1,3 @@
-
 import sys
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, operationerrfmt
@@ -11,6 +10,12 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib import jit
 
+if sys.byteorder == 'little':
+    byteorder_prefix = '<'
+    nonnative_byteorder_prefix = '>'
+else:
+    byteorder_prefix = '>'
+    nonnative_byteorder_prefix = '<'
 
 UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
@@ -44,12 +49,11 @@
     out = base.W_NDimArray.from_shape(space, shape, dtype)
     return out
 
-
 class W_Dtype(W_Root):
     _immutable_fields_ = ["itemtype", "num", "kind", "shape"]
 
     def __init__(self, itemtype, num, kind, name, char, w_box_type,
-                 alternate_constructors=[], aliases=[],
+                 alternate_constructors=[], aliases=[], float_type=None,
                  fields=None, fieldnames=None, native=True, shape=[], subdtype=None):
         self.itemtype = itemtype
         self.num = num
@@ -59,10 +63,10 @@
         self.w_box_type = w_box_type
         self.alternate_constructors = alternate_constructors
         self.aliases = aliases
+        self.float_type = float_type
         self.fields = fields
         self.fieldnames = fieldnames
         self.native = native
-        self.float_type = None
         self.shape = list(shape)
         self.subdtype = subdtype
         if not subdtype:
@@ -227,7 +231,7 @@
         return self.kind == SIGNEDLTR
 
     def is_complex_type(self):
-        return False
+        return self.kind == COMPLEXLTR
 
     def is_float_type(self):
         return (self.kind == FLOATINGLTR or self.float_type is not None)
@@ -300,18 +304,6 @@
         fields = space.getitem(w_data, space.wrap(4))
         self.set_fields(space, fields)
 
-class W_ComplexDtype(W_Dtype):
-    def __init__(self, itemtype, num, kind, name, char, w_box_type,
-                 alternate_constructors=[], aliases=[],
-                 fields=None, fieldnames=None, native=True, float_type=None):
-        W_Dtype.__init__(self, itemtype, num, kind, name, char, w_box_type,
-                 alternate_constructors=alternate_constructors, aliases=aliases,
-                 fields=fields, fieldnames=fieldnames, native=native)
-        self.float_type = float_type
-
-    def is_complex_type(self):
-        return True
-
 def dtype_from_list(space, w_lst):
     lst_w = space.listview(w_lst)
     fields = {}
@@ -345,38 +337,6 @@
     raise OperationError(space.w_NotImplementedError, space.wrap(
         "dtype from dict"))
 
-def variable_dtype(space, name):
-    if name[0] in '<>=':
-        name = name[1:]
-    char = name[0]
-    if len(name) == 1:
-        size = 0
-    else:
-        try:
-            size = int(name[1:])
-        except ValueError:
-            raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
-    if char == 'S' or char == 'c':
-        itemtype = types.StringType(size)
-        basename = 'string'
-        num = 18
-        w_box_type = space.gettypefor(interp_boxes.W_StringBox)
-    elif char == 'V':
-        num = 20
-        basename = 'void'
-        itemtype = types.VoidType(size)
-        return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(size),
-                    "V", space.gettypefor(interp_boxes.W_VoidBox))
-    else:
-        assert char == 'U'
-        basename = 'unicode'
-        itemtype = types.UnicodeType(size)
-        num = 19
-        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
-    return W_Dtype(itemtype, num, char,
-                   basename + str(8 * itemtype.get_element_size()),
-                   char, w_box_type)
-
 def dtype_from_spec(space, name):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "dtype from spec"))
@@ -460,12 +420,38 @@
 )
 W_Dtype.typedef.acceptable_as_base_class = False
 
-if sys.byteorder == 'little':
-    byteorder_prefix = '<'
-    nonnative_byteorder_prefix = '>'
-else:
-    byteorder_prefix = '>'
-    nonnative_byteorder_prefix = '<'
+
+def variable_dtype(space, name):
+    if name[0] in '<>=':
+        name = name[1:]
+    char = name[0]
+    if len(name) == 1:
+        size = 0
+    else:
+        try:
+            size = int(name[1:])
+        except ValueError:
+            raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+    if char == 'S' or char == 'c':
+        itemtype = types.StringType(size)
+        basename = 'string'
+        num = 18
+        w_box_type = space.gettypefor(interp_boxes.W_StringBox)
+    elif char == 'V':
+        num = 20
+        basename = 'void'
+        itemtype = types.VoidType(size)
+        return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(size),
+                    "V", space.gettypefor(interp_boxes.W_VoidBox))
+    else:
+        assert char == 'U'
+        basename = 'unicode'
+        itemtype = types.UnicodeType(size)
+        num = 19
+        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
+    return W_Dtype(itemtype, num, char,
+                   basename + str(8 * itemtype.get_element_size()),
+                   char, w_box_type)
 
 def new_string_dtype(space, size):
     return W_Dtype(
@@ -617,7 +603,7 @@
             w_box_type=space.gettypefor(interp_boxes.W_FloatLongBox),
             aliases=["longdouble", "longfloat"],
         )
-        self.w_complex64dtype = W_ComplexDtype(
+        self.w_complex64dtype = W_Dtype(
             types.Complex64(),
             num=14,
             kind=COMPLEXLTR,
@@ -626,7 +612,7 @@
             w_box_type = space.gettypefor(interp_boxes.W_Complex64Box),
             float_type = self.w_float32dtype,
         )
-        self.w_complex128dtype = W_ComplexDtype(
+        self.w_complex128dtype = W_Dtype(
             types.Complex128(),
             num=15,
             kind=COMPLEXLTR,
@@ -637,7 +623,7 @@
             aliases=["complex"],
             float_type = self.w_float64dtype,
         )
-        self.w_complexlongdtype = W_ComplexDtype(
+        self.w_complexlongdtype = W_Dtype(
             types.ComplexLong(),
             num=16,
             kind=COMPLEXLTR,

From noreply at buildbot.pypy.org  Fri Oct 18 07:46:29 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 18 Oct 2013 07:46:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix creation of dtype('c#')
Message-ID: <20131018054629.AE3A71C0842@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67475:1b0b29ad2a80
Date: 2013-10-18 01:42 -0400
http://bitbucket.org/pypy/pypy/changeset/1b0b29ad2a80/

Log:	fix creation of dtype('c#')

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -432,7 +432,10 @@
             size = int(name[1:])
         except ValueError:
             raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
-    if char == 'S' or char == 'c':
+    if char == 'c':
+        char = 'S'
+        size = 1
+    if char == 'S':
         itemtype = types.StringType(size)
         basename = 'string'
         num = 18
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -800,6 +800,14 @@
         assert d.type is str_
         assert d.name == "string64"
         assert d.num == 18
+        for i in [1, 2, 3]:
+            d = dtype('c%d' % i)
+            assert d.itemsize == 1
+            assert d.kind == 'S'
+            assert d.type is str_
+            assert d.name == 'string8'
+            assert d.num == 18
+            assert d.str == '|S1'
 
     def test_unicode_dtype(self):
         from numpypy import dtype, unicode_

From noreply at buildbot.pypy.org  Fri Oct 18 08:16:11 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 18 Oct 2013 08:16:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: size goes on the name not char
Message-ID: <20131018061611.8A3E01C01B0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67476:e1560a7705ae
Date: 2013-10-18 02:11 -0400
http://bitbucket.org/pypy/pypy/changeset/e1560a7705ae/

Log:	size goes on the name not char

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -457,22 +457,24 @@
                    char, w_box_type)
 
 def new_string_dtype(space, size):
+    itemtype = types.StringType(size)
     return W_Dtype(
-        types.StringType(size),
+        itemtype,
         num=18,
         kind=STRINGLTR,
-        name='string',
-        char='S' + str(size),
+        name='string' + str(8 * itemtype.get_element_size()),
+        char='S',
         w_box_type = space.gettypefor(interp_boxes.W_StringBox),
     )
 
 def new_unicode_dtype(space, size):
+    itemtype = types.UnicodeType(size)
     return W_Dtype(
-        types.UnicodeType(size),
+        itemtype,
         num=19,
         kind=UNICODELTR,
-        name='unicode',
-        char='U' + str(size),
+        name='unicode' + str(8 * itemtype.get_element_size()),
+        char='U',
         w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
     )
 

From noreply at buildbot.pypy.org  Fri Oct 18 11:10:20 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 18 Oct 2013 11:10:20 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: adapt tests to stm in
	test_stmrewrite.py
Message-ID: <20131018091020.77DF61C01F7@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67477:80d972bc755a
Date: 2013-10-18 10:59 +0200
http://bitbucket.org/pypy/pypy/changeset/80d972bc755a/

Log:	adapt tests to stm in test_stmrewrite.py

diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -872,31 +872,31 @@
             p0 = new(descr=sdescr)
             p1 = new(descr=tdescr)
             p2 = new(descr=sdescr)
-            jump()
         """, """
             []
             p0 = call_malloc_nursery(   \
                                %(sdescr.size + tdescr.size + sdescr.size)d)
             setfield_gc(p0, 1234, descr=tiddescr)
+            stm_set_revision_gc(p0, descr=revdescr)
             p1 = int_add(p0, %(sdescr.size)d)
             setfield_gc(p1, 5678, descr=tiddescr)
+            stm_set_revision_gc(p1, descr=revdescr)
             p2 = int_add(p1, %(tdescr.size)d)
             setfield_gc(p2, 1234, descr=tiddescr)
-            jump()
+            stm_set_revision_gc(p2, descr=revdescr)
         """)
 
     def test_rewrite_assembler_new_array_fixed_to_malloc(self):
         self.check_rewrite("""
             []
             p0 = new_array(10, descr=adescr)
-            jump()
         """, """
             []
             p0 = call_malloc_nursery(    \
                                 %(adescr.basesize + 10 * adescr.itemsize)d)
             setfield_gc(p0, 4321, descr=tiddescr)
+            stm_set_revision_gc(p0, descr=revdescr)
             setfield_gc(p0, 10, descr=alendescr)
-            jump()
         """)
 
     def test_rewrite_assembler_new_and_new_array_fixed_to_malloc(self):
@@ -904,30 +904,29 @@
             []
             p0 = new(descr=sdescr)
             p1 = new_array(10, descr=adescr)
-            jump()
         """, """
             []
             p0 = call_malloc_nursery(                                  \
                                 %(sdescr.size +                        \
                                   adescr.basesize + 10 * adescr.itemsize)d)
             setfield_gc(p0, 1234, descr=tiddescr)
+            stm_set_revision_gc(p0, descr=revdescr)
             p1 = int_add(p0, %(sdescr.size)d)
             setfield_gc(p1, 4321, descr=tiddescr)
+            stm_set_revision_gc(p1, descr=revdescr)
             setfield_gc(p1, 10, descr=alendescr)
-            jump()
         """)
 
     def test_rewrite_assembler_round_up(self):
         self.check_rewrite("""
             []
             p0 = new_array(6, descr=bdescr)
-            jump()
         """, """
             []
             p0 = call_malloc_nursery(%(bdescr.basesize + 8)d)
             setfield_gc(p0, 8765, descr=tiddescr)
+            stm_set_revision_gc(p0, descr=revdescr)
             setfield_gc(p0, 6, descr=blendescr)
-            jump()
         """)
 
     def test_rewrite_assembler_round_up_always(self):
@@ -937,22 +936,24 @@
             p1 = new_array(5, descr=bdescr)
             p2 = new_array(5, descr=bdescr)
             p3 = new_array(5, descr=bdescr)
-            jump()
         """, """
             []
             p0 = call_malloc_nursery(%(4 * (bdescr.basesize + 8))d)
             setfield_gc(p0, 8765, descr=tiddescr)
+            stm_set_revision_gc(p0, descr=revdescr)
             setfield_gc(p0, 5, descr=blendescr)
             p1 = int_add(p0, %(bdescr.basesize + 8)d)
             setfield_gc(p1, 8765, descr=tiddescr)
+            stm_set_revision_gc(p1, descr=revdescr)
             setfield_gc(p1, 5, descr=blendescr)
             p2 = int_add(p1, %(bdescr.basesize + 8)d)
             setfield_gc(p2, 8765, descr=tiddescr)
+            stm_set_revision_gc(p2, descr=revdescr)
             setfield_gc(p2, 5, descr=blendescr)
             p3 = int_add(p2, %(bdescr.basesize + 8)d)
             setfield_gc(p3, 8765, descr=tiddescr)
+            stm_set_revision_gc(p3, descr=revdescr)
             setfield_gc(p3, 5, descr=blendescr)
-            jump()
         """)
 
     def test_rewrite_assembler_minimal_size(self):
@@ -960,14 +961,14 @@
             []
             p0 = new(descr=edescr)
             p1 = new(descr=edescr)
-            jump()
         """, """
             []
             p0 = call_malloc_nursery(%(4*WORD)d)
             setfield_gc(p0, 9000, descr=tiddescr)
+            stm_set_revision_gc(p0, descr=revdescr)
             p1 = int_add(p0, %(2*WORD)d)
             setfield_gc(p1, 9000, descr=tiddescr)
-            jump()
+            stm_set_revision_gc(p1, descr=revdescr)
         """)
 
     def test_rewrite_assembler_variable_size(self):
@@ -979,6 +980,7 @@
             [i0]
             p0 = call_malloc_nursery_varsize(0, 1, i0, descr=bdescr)
             setfield_gc(p0, i0, descr=blendescr)
+            stm_transaction_break()
             jump(i0)
         """)
 
@@ -991,6 +993,7 @@
         [i0]
         p0 = call_malloc_nursery_varsize(1, 1, i0, descr=strdescr)
         setfield_gc(p0, i0, descr=strlendescr)
+        stm_transaction_break()
         jump(i0)
         """)
 
@@ -1014,6 +1017,7 @@
                                 %(nonstd_descr.lendescr.offset)d,     \
                                 6464, i0,                             \
                                 descr=malloc_array_nonstandard_descr)
+            stm_transaction_break()
             jump(i0)
         """, nonstd_descr=nonstd_descr)
 
@@ -1028,6 +1032,7 @@
             p0 = call_malloc_gc(ConstClass(malloc_array), 1,  \
                                 %(bdescr.tid)d, 103,          \
                                 descr=malloc_array_descr)
+            stm_transaction_break()
             jump()
         """)
 
@@ -1038,21 +1043,22 @@
             p0 = new_array(101, descr=bdescr)
             p1 = new_array(102, descr=bdescr)  # two new_arrays can be combined
             p2 = new_array(103, descr=bdescr)  # but not all three
-            jump()
         """, """
             []
             p0 = call_malloc_nursery(    \
                               %(2 * (bdescr.basesize + 104))d)
             setfield_gc(p0, 8765, descr=tiddescr)
+            stm_set_revision_gc(p0, descr=revdescr)
             setfield_gc(p0, 101, descr=blendescr)
             p1 = int_add(p0, %(bdescr.basesize + 104)d)
             setfield_gc(p1, 8765, descr=tiddescr)
+            stm_set_revision_gc(p1, descr=revdescr)
             setfield_gc(p1, 102, descr=blendescr)
             p2 = call_malloc_nursery(    \
                               %(bdescr.basesize + 104)d)
             setfield_gc(p2, 8765, descr=tiddescr)
+            stm_set_revision_gc(p2, descr=revdescr)
             setfield_gc(p2, 103, descr=blendescr)
-            jump()
         """)
 
     def test_rewrite_assembler_huge_size(self):
@@ -1066,6 +1072,7 @@
             p0 = call_malloc_gc(ConstClass(malloc_array), 1, \
                                 %(bdescr.tid)d, 20000000,    \
                                 descr=malloc_array_descr)
+            stm_transaction_break()
             jump()
         """)
 
@@ -1073,13 +1080,12 @@
         self.check_rewrite("""
             []
             p0 = new_with_vtable(ConstClass(o_vtable))
-            jump()
         """, """
             [p1]
             p0 = call_malloc_nursery(104)      # rounded up
             setfield_gc(p0, 9315, descr=tiddescr)
+            stm_set_revision_gc(p0, descr=revdescr)
             setfield_gc(p0, ConstClass(o_vtable), descr=vtable_descr)
-            jump()
         """)
 
     def test_new_with_vtable_too_big(self):
@@ -1087,13 +1093,11 @@
         self.check_rewrite("""
             []
             p0 = new_with_vtable(ConstClass(o_vtable))
-            jump()
         """, """
             [p1]
             p0 = call_malloc_gc(ConstClass(malloc_big_fixedsize), 104, 9315, \
                                 descr=malloc_big_fixedsize_descr)
             setfield_gc(p0, ConstClass(o_vtable), descr=vtable_descr)
-            jump()
         """)
 
     def test_rewrite_assembler_newstr_newunicode(self):
@@ -1103,16 +1107,17 @@
             p1 = newunicode(10)
             p2 = newunicode(i2)
             p3 = newstr(i2)
-            jump()
         """, """
             [i2]
             p0 = call_malloc_nursery(                                \
                       %(strdescr.basesize + 16 * strdescr.itemsize + \
                         unicodedescr.basesize + 10 * unicodedescr.itemsize)d)
             setfield_gc(p0, %(strdescr.tid)d, descr=tiddescr)
+            stm_set_revision_gc(p0, descr=revdescr)
             setfield_gc(p0, 14, descr=strlendescr)
             p1 = int_add(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d)
             setfield_gc(p1, %(unicodedescr.tid)d, descr=tiddescr)
+            stm_set_revision_gc(p1, descr=revdescr)
             setfield_gc(p1, 10, descr=unicodelendescr)
             p2 = call_malloc_nursery_varsize(2, 4, i2, \
                                 descr=unicodedescr)
@@ -1120,6 +1125,64 @@
             p3 = call_malloc_nursery_varsize(1, 1, i2, \
                                 descr=strdescr)
             setfield_gc(p3, i2, descr=strlendescr)
-            jump()
         """)
 
+    def test_label_makes_size_unknown(self):
+        self.check_rewrite("""
+            [i2, p3]
+            p1 = new_array(5, descr=cdescr)
+            label(p1, i2, p3)
+            setarrayitem_gc(p1, i2, p3, descr=cdescr)
+        """, """
+            [i2, p3]
+            p1 = call_malloc_nursery(    \
+                                %(cdescr.basesize + 5 * cdescr.itemsize)d)
+            setfield_gc(p1, 8111, descr=tiddescr)
+            stm_set_revision_gc(p1, descr=revdescr)
+            setfield_gc(p1, 5, descr=clendescr)
+            label(p1, i2, p3)
+            cond_call_stm_b(p1, descr=P2Wdescr)
+            setarrayitem_gc(p1, i2, p3, descr=cdescr)
+        """)
+
+    def test_transaction_break_makes_size_unknown(self):
+        class fakeextrainfo:
+            def call_needs_inevitable(self):
+                return False
+        T = rffi.CArrayPtr(rffi.TIME_T)
+        calldescr2 = get_call_descr(self.gc_ll_descr, [T], rffi.TIME_T,
+                                    fakeextrainfo())
+
+        self.gc_ll_descr.max_size_of_young_obj = 300
+        self.check_rewrite("""
+            [i0, f0]
+            p0 = new_array(5, descr=bdescr)
+            p1 = new_array(5, descr=bdescr)
+            call_may_force(123, descr=calldescr2)
+            guard_not_forced() []
+            p2 = new_array(5, descr=bdescr)
+        """, """
+            [i0, f0]
+            p0 = call_malloc_nursery(    \
+                              %(2 * (bdescr.basesize + 8))d)
+            setfield_gc(p0, 8765, descr=tiddescr)
+            stm_set_revision_gc(p0, descr=revdescr)
+            setfield_gc(p0, 5, descr=blendescr)
+            p1 = int_add(p0, %(bdescr.basesize + 8)d)
+            setfield_gc(p1, 8765, descr=tiddescr)
+            stm_set_revision_gc(p1, descr=revdescr)
+            setfield_gc(p1, 5, descr=blendescr)
+            
+            call_may_force(123, descr=calldescr2)
+            guard_not_forced() []
+            stm_transaction_break()
+            
+            p2 = call_malloc_nursery(    \
+                              %(bdescr.basesize + 8)d)
+            setfield_gc(p2, 8765, descr=tiddescr)
+            stm_set_revision_gc(p2, descr=revdescr)
+            setfield_gc(p2, 5, descr=blendescr)
+        """, calldescr2=calldescr2)
+        
+
+

From noreply at buildbot.pypy.org  Fri Oct 18 18:33:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 18 Oct 2013 18:33:19 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: Fix
Message-ID: <20131018163319.8D6511D23DA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67478:5b45d338fca9
Date: 2013-10-18 18:31 +0200
http://bitbucket.org/pypy/pypy/changeset/5b45d338fca9/

Log:	Fix

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -1108,7 +1108,7 @@
     r = lltype.malloc(ELEM.TO)
     r.item0 = recast(ELEM.TO.item0, entry.key)
     r.item1 = recast(ELEM.TO.item1, entry.value)
-    _ll_dict_del(dic, r_uint(i))
+    _ll_dict_del(dic, i)
     return r
 
 def ll_dict_pop(dic, key):

From noreply at buildbot.pypy.org  Sun Oct 20 12:11:52 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 20 Oct 2013 12:11:52 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: Minor fix
Message-ID: <20131020101152.BBCBA1C01B0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rdict-experiments-3
Changeset: r67479:6411cbf93168
Date: 2013-10-20 12:00 +0200
http://bitbucket.org/pypy/pypy/changeset/6411cbf93168/

Log:	Minor fix

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -1085,11 +1085,11 @@
 
     entries = dic.entries
 
-    i = dic.num_used_items - 1
     while True:
+        i = dic.num_used_items - 1
         if entries.valid(i):
             break
-        i -= 1
+        dic.num_used_items -= 1
 
     key = entries[i].key
     index = dic.lookup_function(dic, key, entries.hash(i),

From noreply at buildbot.pypy.org  Mon Oct 21 11:45:31 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 21 Oct 2013 11:45:31 +0200 (CEST)
Subject: [pypy-commit] cffi default: Added Homebrew instructions
Message-ID: <20131021094531.6CEEC1C0163@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1376:9cbe860ba46e
Date: 2013-10-21 11:45 +0200
http://bitbucket.org/cffi/cffi/changeset/9cbe860ba46e/

Log:	Added Homebrew instructions

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -101,7 +101,7 @@
 
 * ``python setup.py install`` or ``python setup_base.py install``
   (should work out of the box on Linux or Windows; see below for
-  `MacOS 10.6`_ or `Windows 64`_.)
+  `MacOS X`_ or `Windows 64`_.)
 
 * or you can directly import and use ``cffi``, but if you don't
   compile the ``_cffi_backend`` extension module, it will fall back
@@ -138,10 +138,23 @@
 (user-supplied) instructions for other platforms.
 
 
-MacOS 10.6
-++++++++++
+MacOS X
++++++++
 
-(Thanks Juraj Sukop for this)
+**Homebrew** (Thanks David Griffin for this)
+
+1) Install homebrew: http://brew.sh
+
+2) Run the following commands in a terminal
+
+::
+
+    brew install pkg-config libffi
+    export PKG_CONFIG_PATH=/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig/ # May change with libffi version
+    pip install cffi
+
+
+Aternatively, **on OS/X 10.6** (Thanks Juraj Sukop for this)
 
 For building libffi you can use the default install path, but then, in
 ``setup.py`` you need to change::

From noreply at buildbot.pypy.org  Mon Oct 21 14:15:25 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 21 Oct 2013 14:15:25 +0200 (CEST)
Subject: [pypy-commit] stmgc default: make nursery_current,
 nursery_nextlimit and active real thread-locals
Message-ID: <20131021121525.24C611C01F5@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r538:2c23968e3d8f
Date: 2013-10-21 10:33 +0200
http://bitbucket.org/pypy/stmgc/changeset/2c23968e3d8f/

Log:	make nursery_current, nursery_nextlimit and active real thread-
	locals

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -643,7 +643,7 @@
 
     k = get_rand(11);
     check(p);
-    assert(thread_descriptor->active);
+    assert(*thread_descriptor->active_ref);
 
     if (k < 3)
         p = simple_events(p, _r, _sr);
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -56,7 +56,7 @@
 }
 
 
-
+__thread int stm_active;
 __thread struct tx_descriptor *thread_descriptor = NULL;
 
 /* 'global_cur_time' is normally a multiple of 2, except when we turn
@@ -103,8 +103,8 @@
 {
   /* Assert that we are running a transaction.
    *      Returns True if this transaction is inevitable. */
-  assert(d->active == 1 + !d->setjmp_buf);
-  return d->active == 2;
+  assert(*d->active_ref == 1 + !d->setjmp_buf);
+  return *d->active_ref == 2;
 }
 
 static pthread_mutex_t mutex_inevitable = PTHREAD_MUTEX_INITIALIZER;
@@ -120,7 +120,7 @@
   pthread_mutex_lock(&mutex_inevitable);
   stm_start_sharedlock();
 
-  if (d->active < 0)
+  if (*d->active_ref < 0)
     {
       inev_mutex_release();
       AbortNowIfDelayed();
@@ -705,7 +705,7 @@
     }
 
   struct tx_descriptor *d = thread_descriptor;
-  assert(d->active >= 1);
+  assert(*d->active_ref >= 1);
 
   /* We need the collection_lock for the sequel; this is required notably
      because we're about to edit flags on a protected object.
@@ -889,7 +889,7 @@
 void SpinLoop(int num)
 {
   struct tx_descriptor *d = thread_descriptor;
-  assert(d->active >= 1);
+  assert(*d->active_ref >= 1);
   assert(num < SPINLOOP_REASONS);
   d->num_spinloops[num]++;
   smp_spinloop();
@@ -924,7 +924,7 @@
       assert(!stm_has_got_any_lock(d));
     }
 
-  assert(d->active != 0);
+  assert(*d->active_ref != 0);
   assert(!is_inevitable(d));
   assert(num < ABORT_REASONS);
   d->num_aborts[num]++;
@@ -989,7 +989,7 @@
   SpinLoop(SPLP_ABORT);
 
   /* make the transaction no longer active */
-  d->active = 0;
+  *d->active_ref = 0;
   d->atomic = 0;
 
   /* release the lock */
@@ -1031,22 +1031,22 @@
 
 void AbortTransactionAfterCollect(struct tx_descriptor *d, int reason)
 {
-  if (d->active >= 0)
+  if (*d->active_ref >= 0)
     {
       dprintf(("abort %d after collect!\n", reason));
-      assert(d->active == 1);   /* not 2, which means inevitable */
-      d->active = -reason;
+      assert(*d->active_ref == 1);   /* not 2, which means inevitable */
+      *d->active_ref = -reason;
     }
-  assert(d->active < 0);
+  assert(*d->active_ref < 0);
 }
 
 void AbortNowIfDelayed(void)
 {
   struct tx_descriptor *d = thread_descriptor;
-  if (d->active < 0)
+  if (*d->active_ref < 0)
     {
-      int reason = -d->active;
-      d->active = 1;
+      int reason = -*d->active_ref;
+      *d->active_ref = 1;
       AbortTransaction(reason);
     }
 }
@@ -1074,9 +1074,9 @@
 static void init_transaction(struct tx_descriptor *d)
 {
   assert(d->atomic == 0);
-  assert(d->active == 0);
+  assert(*d->active_ref == 0);
   stm_start_sharedlock();
-  assert(d->active == 0);
+  assert(*d->active_ref == 0);
 
   if (clock_gettime(CLOCK_MONOTONIC, &d->start_real_time) < 0) {
     d->start_real_time.tv_nsec = -1;
@@ -1097,7 +1097,7 @@
 {
   struct tx_descriptor *d = thread_descriptor;
   init_transaction(d);
-  d->active = 1;
+  *d->active_ref = 1;
   d->setjmp_buf = buf;
   d->longjmp_callback = longjmp_callback;
   d->old_thread_local_obj = stm_thread_local_obj;
@@ -1429,7 +1429,7 @@
 {   /* must save roots around this call */
   revision_t cur_time;
   struct tx_descriptor *d = thread_descriptor;
-  assert(d->active >= 1);
+  assert(*d->active_ref >= 1);
   assert(d->atomic == 0);
   dprintf(("CommitTransaction(%p)\n", d));
   spinlock_acquire(d->public_descriptor->collection_lock, 'C');  /*committing*/
@@ -1502,7 +1502,7 @@
 
   spinlock_release(d->public_descriptor->collection_lock);
   d->num_commits++;
-  d->active = 0;
+  *d->active_ref = 0;
   stm_stop_sharedlock();
 
   /* clear the list of callbacks that would have been called
@@ -1516,7 +1516,7 @@
 {
   d->setjmp_buf = NULL;
   d->old_thread_local_obj = NULL;
-  d->active = 2;
+  *d->active_ref = 2;
   d->reads_size_limit_nonatomic = 0;
   update_reads_size_limit(d);
   dprintf(("make_inevitable(%p)\n", d));
@@ -1543,7 +1543,7 @@
 {   /* must save roots around this call */
   revision_t cur_time;
   struct tx_descriptor *d = thread_descriptor;
-  if (d == NULL || d->active != 1)
+  if (d == NULL || *d->active_ref != 1)
     return;  /* I am already inevitable, or not in a transaction at all
                 (XXX statically we should know when we're outside
                 a transaction) */
@@ -1742,6 +1742,9 @@
       assert(d->my_lock & 1);
       assert(d->my_lock >= LOCKED);
       stm_private_rev_num = -d->my_lock;
+      d->active_ref = &stm_active;
+      d->nursery_current_ref = &stm_nursery_current;
+      d->nursery_nextlimit_ref = &stm_nursery_nextlimit;
       d->private_revision_ref = &stm_private_rev_num;
       d->read_barrier_cache_ref = &stm_read_barrier_cache;
       stm_thread_local_obj = NULL;
@@ -1768,7 +1771,7 @@
     revision_t i;
     struct tx_descriptor *d = thread_descriptor;
     assert(d != NULL);
-    assert(d->active == 0);
+    assert(*d->active_ref == 0);
 
     /* our nursery is empty at this point.  The list 'stolen_objects'
        should have been emptied at the previous minor collection and
diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -174,8 +174,8 @@
   unsigned long count_reads;
   unsigned long reads_size_limit;        /* see should_break_tr. */
   unsigned long reads_size_limit_nonatomic;
-  int active;    /* 0 = inactive, 1 = regular, 2 = inevitable,
-                    negative = killed by collection */
+  int *active_ref;    /* 0 = inactive, 1 = regular, 2 = inevitable,
+                         negative = killed by collection */
   struct timespec start_real_time;
   int max_aborts;
   unsigned int num_commits;
@@ -200,6 +200,7 @@
 
 extern __thread struct tx_descriptor *thread_descriptor;
 extern __thread revision_t stm_private_rev_num;
+extern __thread int stm_active;
 extern struct tx_public_descriptor *stm_descriptor_array[];
 extern struct tx_descriptor *stm_tx_head;
 
diff --git a/c4/extra.c b/c4/extra.c
--- a/c4/extra.c
+++ b/c4/extra.c
@@ -25,7 +25,7 @@
 void stm_call_on_abort(void *key, void callback(void *))
 {
     struct tx_descriptor *d = thread_descriptor;
-    if (d == NULL || d->active != 1)
+    if (d == NULL || *d->active_ref != 1)
         return;   /* ignore callbacks if we're outside a transaction or
                      in an inevitable transaction (which cannot abort) */
     if (callback == NULL) {
@@ -48,7 +48,7 @@
 void stm_invoke_callbacks_on_abort(struct tx_descriptor *d)
 {
     wlog_t *item;
-    assert(d->active == 0);
+    assert(*d->active_ref == 0);
 
     G2L_LOOP_FORWARD(d->callbacks_on_abort, item) {
         void *key = (void *)item->addr;
@@ -286,7 +286,7 @@
         output->signature_packed = 127;
         output->elapsed_time = elapsed_time;
         output->abort_reason = abort_reason;
-        output->active = d->active;
+        output->active = *d->active_ref;
         output->atomic = d->atomic;
         output->count_reads = d->count_reads;
         output->reads_size_limit_nonatomic = d->reads_size_limit_nonatomic;
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -649,7 +649,7 @@
     /* If we're aborting this transaction anyway, we don't need to do
      * more here.
      */
-    if (d->active < 0) {
+    if (*d->active_ref < 0) {
         /* already "aborted" during forced minor collection
            clear list of read objects so that a possible minor collection 
            before the abort doesn't trip 
@@ -659,7 +659,7 @@
         return;
     }
 
-    if (d->active == 2) {
+    if (*d->active_ref == 2) {
         /* inevitable transaction: clear the list of read objects */
         gcptrlist_clear(&d->list_of_read_objects);
     }
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -1,5 +1,9 @@
 #include "stmimpl.h"
 
+
+__thread char *stm_nursery_current;
+__thread char *stm_nursery_nextlimit;
+
 int stmgc_is_in_nursery(struct tx_descriptor *d, gcptr obj)
 {
     return (d->nursery_base <= (char*)obj && ((char*)obj) < d->nursery_end);
@@ -32,8 +36,8 @@
     assert(d->nursery_base == NULL);
     d->nursery_base = stm_malloc(GC_NURSERY);       /* start of nursery */
     d->nursery_end = d->nursery_base + GC_NURSERY;  /* end of nursery */
-    d->nursery_current = d->nursery_base;           /* current position */
-    d->nursery_nextlimit = d->nursery_base;         /* next section limit */
+    *d->nursery_current_ref = d->nursery_base;           /* current position */
+    *d->nursery_nextlimit_ref = d->nursery_base;         /* next section limit */
     d->nursery_cleared = NC_REGULAR;
 
     dprintf(("minor: nursery is at [%p to %p]\n", d->nursery_base,
@@ -48,7 +52,7 @@
        this assert (committransaction() -> 
        updatechainheads() -> stub_malloc() -> ...): */
     assert(!minor_collect_anything_to_do(d)
-           || d->nursery_current == d->nursery_end);
+           || *d->nursery_current_ref == d->nursery_end);
     stm_free(d->nursery_base);
 
     gcptrlist_delete(&d->old_objects_to_trace);
@@ -59,7 +63,7 @@
 void stmgc_minor_collect_soon(void)
 {
     struct tx_descriptor *d = thread_descriptor;
-    d->nursery_current = d->nursery_end;
+    *d->nursery_current_ref = d->nursery_end;
 }
 
 inline static gcptr allocate_nursery(size_t size, revision_t tid)
@@ -67,11 +71,11 @@
     /* if 'tid == -1', we must not collect */
     struct tx_descriptor *d = thread_descriptor;
     gcptr P;
-    char *cur = d->nursery_current;
+    char *cur = *d->nursery_current_ref;
     char *end = cur + size;
     assert((size & 3) == 0);
-    d->nursery_current = end;
-    if (end > d->nursery_nextlimit) {
+    *d->nursery_current_ref = end;
+    if (end > *d->nursery_nextlimit_ref) {
         P = allocate_next_section(size, tid);
     }
     else {
@@ -94,7 +98,7 @@
 {
     /* XXX inline the fast path */
     assert(tid == (tid & STM_USER_TID_MASK));
-    assert(thread_descriptor->active > 0);
+    assert(*thread_descriptor->active_ref > 0);
     gcptr P = allocate_nursery(size, tid);
     P->h_revision = stm_private_rev_num;
     assert(P->h_original == 0);  /* null-initialized already */
@@ -402,7 +406,7 @@
     gcptr *items = d->list_of_read_objects.items;
     assert(d->list_of_read_objects.size >= limit);
 
-    if (d->active == 2) {
+    if (*d->active_ref == 2) {
         /* inevitable transaction: clear the list of read objects */
         gcptrlist_clear(&d->list_of_read_objects);
     }
@@ -501,10 +505,10 @@
        Second, if the thread is really idle, then its nursery is sent
        back to the system until it's really needed.
     */
-    if ((d->nursery_nextlimit - d->nursery_base) < GC_NURSERY / 10) {
+    if ((*d->nursery_nextlimit_ref - d->nursery_base) < GC_NURSERY / 10) {
         size_t already_cleared = 0;
         if (d->nursery_cleared == NC_ALREADY_CLEARED) {
-            already_cleared = d->nursery_end - d->nursery_current;
+            already_cleared = d->nursery_end - *d->nursery_current_ref;
         }
         stm_clear_large_memory_chunk(d->nursery_base, GC_NURSERY,
                                      already_cleared);
@@ -513,7 +517,7 @@
     else {
         d->nursery_cleared = NC_REGULAR;
 #if defined(_GC_DEBUG)
-        memset(d->nursery_current, 0xEE, d->nursery_end - d->nursery_current);
+        memset(*d->nursery_current_ref, 0xEE, d->nursery_end - *d->nursery_current_ref);
 #endif
     }
 
@@ -531,8 +535,8 @@
     if (d->nursery_cleared == NC_ALREADY_CLEARED)
         memset(d->nursery_base, 0, GC_NURSERY);
 #endif
-    d->nursery_current = d->nursery_base;
-    d->nursery_nextlimit = d->nursery_base;
+    *d->nursery_current_ref = d->nursery_base;
+    *d->nursery_nextlimit_ref = d->nursery_base;
 
     assert(!minor_collect_anything_to_do(d));
 }
@@ -540,7 +544,7 @@
 void stmgc_minor_collect(void)
 {
     struct tx_descriptor *d = thread_descriptor;
-    assert(d->active >= 1);
+    assert(*d->active_ref >= 1);
     minor_collect(d);
     AbortNowIfDelayed();
 }
@@ -554,7 +558,7 @@
 #ifndef NDEBUG
 int minor_collect_anything_to_do(struct tx_descriptor *d)
 {
-    if (d->nursery_current == d->nursery_base /*&&
+    if (*d->nursery_current_ref == d->nursery_base /*&&
         !g2l_any_entry(&d->young_objects_outside_nursery)*/ ) {
         /* there is no young object */
         assert(gcptrlist_size(&d->public_with_young_copy) == 0);
@@ -588,7 +592,7 @@
        First fix 'nursery_current', left to a bogus value by the caller.
     */
     struct tx_descriptor *d = thread_descriptor;
-    d->nursery_current -= allocate_size;
+    *d->nursery_current_ref -= allocate_size;
 
     /* Are we asking for a "reasonable" number of bytes, i.e. a value
        at most equal to one section?
@@ -608,8 +612,8 @@
     }
 
     /* Are we at the end of the nursery? */
-    if (d->nursery_nextlimit == d->nursery_end ||
-        d->nursery_current == d->nursery_end) {   // stmgc_minor_collect_soon()
+    if (*d->nursery_nextlimit_ref == d->nursery_end ||
+        *d->nursery_current_ref == d->nursery_end) {   // stmgc_minor_collect_soon()
         /* Yes */
         if (tid == -1)
             return NULL;    /* cannot collect */
@@ -619,19 +623,19 @@
         stmgc_minor_collect();
         stmgcpage_possibly_major_collect(0);
 
-        assert(d->nursery_current == d->nursery_base);
-        assert(d->nursery_nextlimit == d->nursery_base);
+        assert(*d->nursery_current_ref == d->nursery_base);
+        assert(*d->nursery_nextlimit_ref == d->nursery_base);
     }
 
     /* Clear the next section */
     if (d->nursery_cleared != NC_ALREADY_CLEARED)
-        memset(d->nursery_nextlimit, 0, GC_NURSERY_SECTION);
-    d->nursery_nextlimit += GC_NURSERY_SECTION;
+        memset(*d->nursery_nextlimit_ref, 0, GC_NURSERY_SECTION);
+    *d->nursery_nextlimit_ref += GC_NURSERY_SECTION;
 
     /* Return the object from there */
-    gcptr P = (gcptr)d->nursery_current;
-    d->nursery_current += allocate_size;
-    assert(d->nursery_current <= d->nursery_nextlimit);
+    gcptr P = (gcptr)*d->nursery_current_ref;
+    *d->nursery_current_ref += allocate_size;
+    assert(*d->nursery_current_ref <= *d->nursery_nextlimit_ref);
 
     P->h_tid = tid;
     assert_cleared(((char *)P) + sizeof(revision_t),
diff --git a/c4/nursery.h b/c4/nursery.h
--- a/c4/nursery.h
+++ b/c4/nursery.h
@@ -24,8 +24,8 @@
 
 #define NURSERY_FIELDS_DECL                                             \
     /* the nursery */                                                   \
-    char *nursery_current;                                              \
-    char *nursery_nextlimit;                                            \
+    char **nursery_current_ref;                                         \
+    char **nursery_nextlimit_ref;                                       \
     char *nursery_end;                                                  \
     char *nursery_base;                                                 \
     enum { NC_REGULAR, NC_ALREADY_CLEARED } nursery_cleared;            \
@@ -57,6 +57,9 @@
 
 
 struct tx_descriptor;  /* from et.h */
+extern __thread char *stm_nursery_current;
+extern __thread char *stm_nursery_nextlimit;
+
 
 void stmgc_init_nursery(void);
 void stmgc_done_nursery(void);
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -196,12 +196,14 @@
 /* macro functionality */
 
 extern __thread gcptr *stm_shadowstack;
+extern __thread int stm_active;
+extern __thread char *stm_nursery_current;
+extern __thread char *stm_nursery_nextlimit;
 
 #define stm_push_root(obj)  (*stm_shadowstack++ = (obj))
 #define stm_pop_root()      (*--stm_shadowstack)
 
 extern __thread revision_t stm_private_rev_num;
-extern __thread struct tx_descriptor *thread_descriptor; /* XXX: stm_ prefix */
 gcptr stm_DirectReadBarrier(gcptr);
 gcptr stm_WriteBarrier(gcptr);
 gcptr stm_RepeatReadBarrier(gcptr);
diff --git a/c4/stmsync.c b/c4/stmsync.c
--- a/c4/stmsync.c
+++ b/c4/stmsync.c
@@ -42,7 +42,7 @@
                                    d->reads_size_limit_nonatomic));
     /* if is_inevitable(), reads_size_limit_nonatomic should be 0
        (and thus reads_size_limit too, if !d->atomic.) */
-    if (d->active == 2)
+    if (*d->active_ref == 2)
         assert(d->reads_size_limit_nonatomic == 0);
 #endif
 
@@ -167,7 +167,7 @@
            has configured 'reads_size_limit_nonatomic' to a smaller value.
            When such a shortened transaction succeeds, the next one will
            see its length limit doubled, up to the maximum. */
-        if (counter == 0 && d->active != 2) {
+        if (counter == 0 && *d->active_ref != 2) {
             unsigned long limit = d->reads_size_limit_nonatomic;
             if (limit != 0 && limit < (stm_regular_length_limit >> 1))
                 limit = (limit << 1) | 1;
@@ -182,7 +182,7 @@
             /* atomic transaction: a common case is that callback() returned
                even though we are atomic because we need a major GC.  For
                that case, release and reaquire the rw lock here. */
-            assert(d->active >= 1);
+            assert(*d->active_ref >= 1);
             stm_possible_safe_point();
         }
 
@@ -217,7 +217,7 @@
 {   /* must save roots around this call */
     struct tx_descriptor *d = thread_descriptor;
     if (d->atomic) {
-        assert(d->active >= 1);
+        assert(*d->active_ref >= 1);
         stm_possible_safe_point();
     }
     else {
@@ -266,7 +266,7 @@
 int stm_in_transaction(void)
 {
     struct tx_descriptor *d = thread_descriptor;
-    return d && d->active;
+    return d && *d->active_ref;
 }
 
 /************************************************************/
@@ -353,7 +353,7 @@
 
     /* Warning, may block waiting for rwlock_in_transaction while another
        thread runs a major GC */
-    assert(thread_descriptor->active);
+    assert(*thread_descriptor->active_ref);
     assert(in_single_thread != thread_descriptor);
 
     stm_stop_sharedlock();

From noreply at buildbot.pypy.org  Mon Oct 21 14:47:47 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 21 Oct 2013 14:47:47 +0200 (CEST)
Subject: [pypy-commit] cffi default: Docstring fix
Message-ID: <20131021124747.66A0F1C0149@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1377:4ef315f70817
Date: 2013-10-21 14:23 +0200
http://bitbucket.org/cffi/cffi/changeset/4ef315f70817/

Log:	Docstring fix

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -133,7 +133,7 @@
 
     def typeof(self, cdecl):
         """Parse the C type given as a string and return the
-        corresponding Python type: <class 'ffi.CData<...>'>.
+        corresponding <ctype> object.
         It can also be used on 'cdata' instance to get its C type.
         """
         if isinstance(cdecl, basestring):

From noreply at buildbot.pypy.org  Mon Oct 21 14:47:48 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 21 Oct 2013 14:47:48 +0200 (CEST)
Subject: [pypy-commit] cffi default: Document the workaround for taking
 pointers to functions.
Message-ID: <20131021124748.F19C81C0149@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1378:f37a06c4c060
Date: 2013-10-21 14:31 +0200
http://bitbucket.org/cffi/cffi/changeset/f37a06c4c060/

Log:	Document the workaround for taking pointers to functions.

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -981,6 +981,17 @@
 
 Aside from these limitations, functions and callbacks can return structs.
 
+CPython only: for performance, ``ffi.verify()`` returns functions as
+objects of type ``<built-in function>``.  They are not ``<cdata>``, so
+you cannot e.g. pass them to some other C function expecting a function
+pointer argument.  Only ``ffi.typeof()`` works on them.  If you really
+need a pointer to the function, use the following workaround::
+  
+    ffi.cdef(""" int (*foo)(int a, int b); """)
+
+i.e. declare them as pointer-to-function in the cdef (even if they are
+regular functions in the C code).
+
 
 Variadic function calls
 -----------------------

From noreply at buildbot.pypy.org  Mon Oct 21 15:01:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 21 Oct 2013 15:01:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: Support for test_calling_object_init on
	top of pypy
Message-ID: <20131021130108.C54041C15A7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67480:7f3f5f2569ae
Date: 2013-10-21 15:00 +0200
http://bitbucket.org/pypy/pypy/changeset/7f3f5f2569ae/

Log:	Support for test_calling_object_init on top of pypy

diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -356,7 +356,8 @@
         getattr(WindowsError.__init__, 'im_func', WindowsError.__init__)] = (
         rtype_WindowsError__init__)
 
-BUILTIN_TYPER[object.__init__] = rtype_object__init__
+BUILTIN_TYPER[getattr(object.__init__, 'im_func', object.__init__)] = (
+    rtype_object__init__)
 # annotation of low-level types
 
 def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None,

From noreply at buildbot.pypy.org  Mon Oct 21 16:31:16 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 21 Oct 2013 16:31:16 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: hg merge default
Message-ID: <20131021143116.1ED5E1C02C5@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67481:f43349efb6b7
Date: 2013-10-21 16:30 +0200
http://bitbucket.org/pypy/pypy/changeset/f43349efb6b7/

Log:	hg merge default

diff too long, truncating to 2000 out of 11788 lines

diff --git a/lib_pypy/numpy.py b/lib_pypy/numpy.py
deleted file mode 100644
--- a/lib_pypy/numpy.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import warnings
-import sys
-if 'numpypy' not in sys.modules:
-    warnings.warn(
-        "The 'numpy' module of PyPy is in-development and not complete. "
-        "To avoid this warning, write 'import numpypy as numpy'. ",
-        UserWarning) # XXX is this the best warning type?
-
-from numpypy import *
-import numpypy
-__all__ = numpypy.__all__
-del numpypy
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,24 +1,17 @@
-import core
-from core import *
-import lib
-from lib import *
+from . import core
+from .core import *
+from . import lib
+from .lib import *
 
 from __builtin__ import bool, int, long, float, complex, object, unicode, str
-from core import abs, max, min
+
+from .core import round, abs, max, min
 
 __version__ = '1.7.0'
 
-import os
-def get_include():
-    head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.join(head, '../include')
-
-
-__all__ = ['__version__', 'get_include']
+__all__ = ['__version__']
 __all__ += core.__all__
 __all__ += lib.__all__
 
 #import sys
 #sys.modules.setdefault('numpy', sys.modules['numpypy'])
-
-
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
--- a/lib_pypy/numpypy/core/__init__.py
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -1,12 +1,17 @@
-import numeric
-from numeric import *
-import fromnumeric
-from fromnumeric import *
-import shape_base
-from shape_base import *
+from __future__ import division, absolute_import, print_function
 
-from fromnumeric import amax as max, amin as min
-from numeric import absolute as abs
+from . import multiarray
+from . import umath
+from . import numeric
+from .numeric import *
+from . import fromnumeric
+from .fromnumeric import *
+from . import shape_base
+from .shape_base import *
+
+from .fromnumeric import amax as max, amin as min, \
+    round_ as round
+from .numeric import absolute as abs
 
 __all__ = []
 __all__ += numeric.__all__
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ b/lib_pypy/numpypy/core/arrayprint.py
@@ -247,10 +247,11 @@
     formatdict = {'bool' : _boolFormatter,
                   'int' : IntegerFormat(data),
                   'float' : FloatFormat(data, precision, suppress_small),
-                  'longfloat' : LongFloatFormat(precision),
+                  'longfloat' : FloatFormat(data, precision, suppress_small),
                   'complexfloat' : ComplexFormat(data, precision,
                                                  suppress_small),
-                  'longcomplexfloat' : LongComplexFormat(precision),
+                  'longcomplexfloat' : ComplexFormat(data, precision,
+                                                     suppress_small),
                   'datetime' : DatetimeFormat(data),
                   'timedelta' : TimedeltaFormat(data),
                   'numpystr' : repr_format,
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -1,36 +1,53 @@
-######################################################################    
+######################################################################
 # This is a copy of numpy/core/fromnumeric.py modified for numpypy
 ######################################################################
-# Each name in __all__ was a function in  'numeric' that is now 
-# a method in 'numpy'.
-# When the corresponding method is added to numpypy BaseArray
-# each function should be added as a module function 
-# at the applevel 
-# This can be as simple as doing the following
-#
-# def func(a, ...):
-#     if not hasattr(a, 'func')
-#         a = numpypy.array(a)
-#     return a.func(...)
-#
-######################################################################
-
-import numpypy
-import _numpypy
-
-# Module containing non-deprecated functions borrowed from Numeric.
-__docformat__ = "restructuredtext en"
+"""Module containing non-deprecated functions borrowed from Numeric.
+
+"""
+from __future__ import division, absolute_import, print_function
+
+import types
+
+from . import multiarray as mu
+from . import umath as um
+from . import numerictypes as nt
+from .numeric import asarray, array, asanyarray, concatenate
+from . import _methods
+
+
+# functions that are methods
+__all__ = [
+        'alen', 'all', 'alltrue', 'amax', 'amin', 'any', 'argmax',
+        'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip',
+        'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean',
+        'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put',
+        'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_',
+        'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze',
+        'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var',
+        ]
+
+
+try:
+    _gentype = types.GeneratorType
+except AttributeError:
+    _gentype = type(None)
+
+# save away Python sum
+_sum_ = sum
 
 # functions that are now methods
-__all__ = ['take', 'reshape', 'choose', 'repeat', 'put',
-           'swapaxes', 'transpose', 'sort', 'argsort', 'argmax', 'argmin',
-           'searchsorted', 'alen',
-           'resize', 'diagonal', 'trace', 'ravel', 'nonzero', 'shape',
-           'compress', 'clip', 'sum', 'product', 'prod', 'sometrue', 'alltrue',
-           'any', 'all', 'cumsum', 'cumproduct', 'cumprod', 'ptp', 'ndim',
-           'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze',
-           'amax', 'amin',
-          ]
+def _wrapit(obj, method, *args, **kwds):
+    try:
+        wrap = obj.__array_wrap__
+    except AttributeError:
+        wrap = None
+    result = getattr(asarray(obj), method)(*args, **kwds)
+    if wrap:
+        if not isinstance(result, mu.ndarray):
+            result = asarray(result)
+        result = wrap(result)
+    return result
+
 
 def take(a, indices, axis=None, out=None, mode='raise'):
     """
@@ -46,6 +63,10 @@
         The source array.
     indices : array_like
         The indices of the values to extract.
+
+        .. versionadded:: 1.8.0
+
+        Also allow scalars for indices.
     axis : int, optional
         The axis over which to select values. By default, the flattened
         input array is used.
@@ -85,8 +106,17 @@
     >>> a[indices]
     array([4, 3, 6])
 
+    If `indices` is not one dimensional, the output also has these dimensions.
+
+    >>> np.take(a, [[0, 1], [2, 3]])
+    array([[4, 3],
+           [5, 7]])
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        take = a.take
+    except AttributeError:
+        return _wrapit(a, 'take', indices, axis, out, mode)
+    return take(indices, axis, out, mode)
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -104,16 +134,23 @@
         One shape dimension can be -1. In this case, the value is inferred
         from the length of the array and remaining dimensions.
     order : {'C', 'F', 'A'}, optional
-        Determines whether the array data should be viewed as in C
-        (row-major) order, FORTRAN (column-major) order, or the C/FORTRAN
-        order should be preserved.
+        Read the elements of `a` using this index order, and place the elements
+        into the reshaped array using this index order.  'C' means to
+        read / write the elements using C-like index order, with the last axis index
+        changing fastest, back to the first axis index changing slowest.  'F'
+        means to read / write the elements using Fortran-like index order, with
+        the first index changing fastest, and the last index changing slowest.
+        Note that the 'C' and 'F' options take no account of the memory layout
+        of the underlying array, and only refer to the order of indexing.  'A'
+        means to read / write the elements in Fortran-like index order if `a` is
+        Fortran *contiguous* in memory, C-like order otherwise.
 
     Returns
     -------
     reshaped_array : ndarray
         This will be a new view object if possible; otherwise, it will
-        be a copy.
-
+        be a copy.  Note there is no guarantee of the *memory layout* (C- or
+        Fortran- contiguous) of the returned array.
 
     See Also
     --------
@@ -121,7 +158,6 @@
 
     Notes
     -----
-
     It is not always possible to change the shape of an array without
     copying the data. If you want an error to be raise if the data is copied,
     you should assign the new shape to the shape attribute of the array::
@@ -129,12 +165,39 @@
      >>> a = np.zeros((10, 2))
      # A transpose make the array non-contiguous
      >>> b = a.T
-     # Taking a view makes it possible to modify the shape without modiying the
+     # Taking a view makes it possible to modify the shape without modifying the
      # initial object.
      >>> c = b.view()
      >>> c.shape = (20)
      AttributeError: incompatible shape for a non-contiguous array
 
+    The `order` keyword gives the index ordering both for *fetching* the values
+    from `a`, and then *placing* the values into the output array.  For example,
+    let's say you have an array:
+
+    >>> a = np.arange(6).reshape((3, 2))
+    >>> a
+    array([[0, 1],
+           [2, 3],
+           [4, 5]])
+
+    You can think of reshaping as first raveling the array (using the given
+    index order), then inserting the elements from the raveled array into the
+    new array using the same kind of index ordering as was used for the
+    raveling.
+
+    >>> np.reshape(a, (2, 3)) # C-like index ordering
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(a, (2, 3), order='F') # Fortran-like index ordering
+    array([[0, 4, 3],
+           [2, 1, 5]])
+    >>> np.reshape(np.ravel(a, order='F'), (2, 3), order='F')
+    array([[0, 4, 3],
+           [2, 1, 5]])
 
     Examples
     --------
@@ -148,12 +211,13 @@
     array([[1, 2],
            [3, 4],
            [5, 6]])
-
     """
     assert order == 'C'
-    if not hasattr(a, 'reshape'):
-       a = numpypy.array(a)
-    return a.reshape(newshape)
+    try:
+        reshape = a.reshape
+    except AttributeError:
+        return _wrapit(a, 'reshape', newshape)
+    return reshape(newshape)
 
 
 def choose(a, choices, out=None, mode='raise'):
@@ -275,7 +339,11 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    return _numpypy.choose(a, choices, out, mode)
+    try:
+        choose = a.choose
+    except AttributeError:
+        return _wrapit(a, 'choose', choices, out=out, mode=mode)
+    return choose(choices, out=out, mode=mode)
 
 
 def repeat(a, repeats, axis=None):
@@ -317,7 +385,11 @@
            [3, 4]])
 
     """
-    return _numpypy.repeat(a, repeats, axis)
+    try:
+        repeat = a.repeat
+    except AttributeError:
+        return _wrapit(a, 'repeat', repeats, axis)
+    return repeat(repeats, axis)
 
 
 def put(a, ind, v, mode='raise'):
@@ -368,7 +440,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return a.put(ind, v, mode)
 
 
 def swapaxes(a, axis1, axis2):
@@ -412,7 +484,10 @@
             [3, 7]]])
 
     """
-    swapaxes = a.swapaxes
+    try:
+        swapaxes = a.swapaxes
+    except AttributeError:
+        return _wrapit(a, 'swapaxes', axis1, axis2)
     return swapaxes(axis1, axis2)
 
 
@@ -456,9 +531,158 @@
     """
     if axes is not None:
         raise NotImplementedError('No "axes" arg yet.')
-    if not hasattr(a, 'T'):
-        a = numpypy.array(a)
-    return a.T
+    try:
+        transpose = a.transpose
+    except AttributeError:
+        return _wrapit(a, 'transpose')
+    return transpose()
+
+
+def partition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Return a partitioned copy of an array.
+
+    Creates a copy of the array with its elements rearranged in such a way that
+    the value of the element in kth position is in the position it would be in
+    a sorted array. All elements smaller than the kth element are moved before
+    this element and all equal or greater are moved behind it. The ordering of
+    the elements in the two partitions is undefined.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to be sorted.
+    kth : int or sequence of ints
+        Element index to partition by. The kth value of the element will be in
+        its final sorted position and all smaller elements will be moved before
+        it and all equal or greater elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all elements
+        indexed by kth  of them into their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort. If None, the array is flattened before
+        sorting. The default is -1, which sorts along the last axis.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'.
+    order : list, optional
+        When `a` is a structured array, this argument specifies which fields
+        to compare first, second, and so on.  This list does not need to
+        include all of the fields.
+
+    Returns
+    -------
+    partitioned_array : ndarray
+        Array of the same type and shape as `a`.
+
+    See Also
+    --------
+    ndarray.partition : Method to sort an array in-place.
+    argpartition : Indirect partition.
+    sort : Full sorting
+
+    Notes
+    -----
+    The various selection algorithms are characterized by their average speed,
+    worst case performance, work space size, and whether they are stable. A
+    stable sort keeps items with the same key in the same relative order. The
+    three available algorithms have the following properties:
+
+    ================= ======= ============= ============ =======
+       kind            speed   worst case    work space  stable
+    ================= ======= ============= ============ =======
+    'introselect'        1        O(n)           0         no
+    ================= ======= ============= ============ =======
+
+    All the partition algorithms make temporary copies of the data when
+    partitioning along any but the last axis.  Consequently, partitioning
+    along the last axis is faster and uses less space than partitioning
+    along any other axis.
+
+    The sort order for complex numbers is lexicographic. If both the real
+    and imaginary parts are non-nan then the order is determined by the
+    real parts except when they are equal, in which case the order is
+    determined by the imaginary parts.
+
+    Examples
+    --------
+    >>> a = np.array([3, 4, 2, 1])
+    >>> np.partition(a, 3)
+    array([2, 1, 3, 4])
+
+    >>> np.partition(a, (1, 3))
+    array([1, 2, 3, 4])
+
+    """
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.partition(kth, axis=axis, kind=kind, order=order)
+    return a
+
+
+def argpartition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Perform an indirect partition along the given axis using the algorithm
+    specified by the `kind` keyword. It returns an array of indices of the
+    same shape as `a` that index data along the given axis in partitioned
+    order.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to sort.
+    kth : int or sequence of ints
+        Element index to partition by. The kth element will be in its final
+        sorted position and all smaller elements will be moved before it and
+        all larger elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all of them into
+        their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort.  The default is -1 (the last axis). If None,
+        the flattened array is used.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'
+    order : list, optional
+        When `a` is an array with fields defined, this argument specifies
+        which fields to compare first, second, etc.  Not all fields need be
+        specified.
+
+    Returns
+    -------
+    index_array : ndarray, int
+        Array of indices that partition `a` along the specified axis.
+        In other words, ``a[index_array]`` yields a sorted `a`.
+
+    See Also
+    --------
+    partition : Describes partition algorithms used.
+    ndarray.partition : Inplace partition.
+    argsort : Full indirect sort
+
+    Notes
+    -----
+    See `partition` for notes on the different selection algorithms.
+
+    Examples
+    --------
+    One dimensional array:
+
+    >>> x = np.array([3, 4, 2, 1])
+    >>> x[np.argpartition(x, 3)]
+    array([2, 1, 3, 4])
+    >>> x[np.argpartition(x, (1, 3))]
+    array([1, 2, 3, 4])
+
+    """
+    return a.argpartition(kth, axis, kind=kind, order=order)
+
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -489,6 +713,7 @@
     argsort : Indirect sort.
     lexsort : Indirect stable sort on multiple keys.
     searchsorted : Find elements in a sorted array.
+    partition : Partial sort.
 
     Notes
     -----
@@ -559,7 +784,13 @@
           dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.sort(axis, kind, order)
+    return a
 
 
 def argsort(a, axis=-1, kind='quicksort', order=None):
@@ -595,6 +826,7 @@
     sort : Describes sorting algorithms used.
     lexsort : Indirect stable sort with multiple keys.
     ndarray.sort : Inplace sort.
+    argpartition : Indirect partial sort.
 
     Notes
     -----
@@ -640,7 +872,11 @@
     array([0, 1])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        argsort = a.argsort
+    except AttributeError:
+        return _wrapit(a, 'argsort', axis, kind, order)
+    return argsort(axis, kind, order)
 
 
 def argmax(a, axis=None):
@@ -694,9 +930,11 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmax'):
-        a = numpypy.array(a)
-    return a.argmax()
+    try:
+        argmax = a.argmax
+    except AttributeError:
+        return _wrapit(a, 'argmax')
+    return argmax()
 
 
 def argmin(a, axis=None):
@@ -710,29 +948,37 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmin'):
-        a = numpypy.array(a)
-    return a.argmin()
-
-
-def searchsorted(a, v, side='left'):
+    try:
+        argmin = a.argmin
+    except AttributeError:
+        return _wrapit(a, 'argmin')
+    return argmin()
+
+
+def searchsorted(a, v, side='left', sorter=None):
     """
     Find indices where elements should be inserted to maintain order.
 
-    Find the indices into a sorted array `a` such that, if the corresponding
-    elements in `v` were inserted before the indices, the order of `a` would
-    be preserved.
+    Find the indices into a sorted array `a` such that, if the
+    corresponding elements in `v` were inserted before the indices, the
+    order of `a` would be preserved.
 
     Parameters
     ----------
     a : 1-D array_like
-        Input array, sorted in ascending order.
+        Input array. If `sorter` is None, then it must be sorted in
+        ascending order, otherwise `sorter` must be an array of indices
+        that sort it.
     v : array_like
         Values to insert into `a`.
     side : {'left', 'right'}, optional
-        If 'left', the index of the first suitable location found is given.  If
-        'right', return the last such index.  If there is no suitable
+        If 'left', the index of the first suitable location found is given.
+        If 'right', return the last such index.  If there is no suitable
         index, return either 0 or N (where N is the length of `a`).
+    sorter : 1-D array_like, optional
+        .. versionadded:: 1.7.0
+        Optional array of integer indices that sort array a into ascending
+        order. They are typically the result of argsort.
 
     Returns
     -------
@@ -761,7 +1007,11 @@
     array([0, 5, 1, 2])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        searchsorted = a.searchsorted
+    except AttributeError:
+        return _wrapit(a, 'searchsorted', v, side, sorter)
+    return searchsorted(v, side, sorter)
 
 
 def resize(a, new_shape):
@@ -802,10 +1052,30 @@
            [0, 1, 2, 3]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def squeeze(a):
+    if isinstance(new_shape, (int, nt.integer)):
+        new_shape = (new_shape,)
+    a = ravel(a)
+    Na = len(a)
+    if not Na: return mu.zeros(new_shape, a.dtype.char)
+    total_size = um.multiply.reduce(new_shape)
+    n_copies = int(total_size / Na)
+    extra = total_size % Na
+
+    if total_size == 0:
+        return a[:0]
+
+    if extra != 0:
+        n_copies = n_copies+1
+        extra = Na-extra
+
+    a = concatenate( (a,)*n_copies)
+    if extra > 0:
+        a = a[:-extra]
+
+    return reshape(a, new_shape)
+
+
+def squeeze(a, axis=None):
     """
     Remove single-dimensional entries from the shape of an array.
 
@@ -813,12 +1083,19 @@
     ----------
     a : array_like
         Input data.
+    axis : None or int or tuple of ints, optional
+        .. versionadded:: 1.7.0
+
+        Selects a subset of the single-dimensional entries in the
+        shape. If an axis is selected with shape entry greater than
+        one, an error is raised.
 
     Returns
     -------
     squeezed : ndarray
-        The input array, but with with all dimensions of length 1
-        removed.  Whenever possible, a view on `a` is returned.
+        The input array, but with with all or a subset of the
+        dimensions of length 1 removed. This is always `a` itself
+        or a view into `a`.
 
     Examples
     --------
@@ -827,9 +1104,20 @@
     (1, 3, 1)
     >>> np.squeeze(x).shape
     (3,)
+    >>> np.squeeze(x, axis=(2,)).shape
+    (1, 3)
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        squeeze = a.squeeze
+    except AttributeError:
+        return _wrapit(a, 'squeeze')
+    try:
+        # First try to use the new axis= parameter
+        return squeeze(axis=axis)
+    except TypeError:
+        # For backwards compatibility
+        return squeeze()
 
 
 def diagonal(a, offset=0, axis1=0, axis2=1):
@@ -844,6 +1132,27 @@
     removing `axis1` and `axis2` and appending an index to the right equal
     to the size of the resulting diagonals.
 
+    In versions of NumPy prior to 1.7, this function always returned a new,
+    independent array containing a copy of the values in the diagonal.
+
+    In NumPy 1.7 and 1.8, it continues to return a copy of the diagonal,
+    but depending on this fact is deprecated. Writing to the resulting
+    array continues to work as it used to, but a FutureWarning is issued.
+
+    In NumPy 1.9 it returns a read-only view on the original array.
+    Attempting to write to the resulting array will produce an error.
+
+    In NumPy 1.10, it will return a read/write view, Writing to the returned
+    array will alter your original array.
+
+    If you don't write to the array returned by this function, then you can
+    just ignore all of the above.
+
+    If you depend on the current behavior, then we suggest copying the
+    returned array explicitly, i.e., use ``np.diagonal(a).copy()`` instead of
+    just ``np.diagonal(a)``. This will work with both past and future versions
+    of NumPy.
+
     Parameters
     ----------
     a : array_like
@@ -913,7 +1222,7 @@
            [5, 7]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return asarray(a).diagonal(offset, axis1, axis2)
 
 
 def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None):
@@ -972,7 +1281,7 @@
     (2, 3)
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return asarray(a).trace(offset, axis1, axis2, dtype, out)
 
 def ravel(a, order='C'):
     """
@@ -984,21 +1293,25 @@
     Parameters
     ----------
     a : array_like
-        Input array.  The elements in ``a`` are read in the order specified by
+        Input array.  The elements in `a` are read in the order specified by
         `order`, and packed as a 1-D array.
     order : {'C','F', 'A', 'K'}, optional
-        The elements of ``a`` are read in this order. 'C' means to view
-        the elements in C (row-major) order. 'F' means to view the elements
-        in Fortran (column-major) order. 'A' means to view the elements
-        in 'F' order if a is Fortran contiguous, 'C' order otherwise.
-        'K' means to view the elements in the order they occur in memory,
-        except for reversing the data when strides are negative.
-        By default, 'C' order is used.
+        The elements of `a` are read using this index order. 'C' means to
+        index the elements in C-like order, with the last axis index changing
+        fastest, back to the first axis index changing slowest.   'F' means to
+        index the elements in Fortran-like index order, with the first index
+        changing fastest, and the last index changing slowest. Note that the 'C'
+        and 'F' options take no account of the memory layout of the underlying
+        array, and only refer to the order of axis indexing.  'A' means to read
+        the elements in Fortran-like index order if `a` is Fortran *contiguous*
+        in memory, C-like order otherwise.  'K' means to read the elements in
+        the order they occur in memory, except for reversing the data when
+        strides are negative.  By default, 'C' index order is used.
 
     Returns
     -------
     1d_array : ndarray
-        Output of the same dtype as `a`, and of shape ``(a.size(),)``.
+        Output of the same dtype as `a`, and of shape ``(a.size,)``.
 
     See Also
     --------
@@ -1008,11 +1321,11 @@
 
     Notes
     -----
-    In row-major order, the row index varies the slowest, and the column
-    index the quickest.  This can be generalized to multiple dimensions,
-    where row-major order implies that the index along the first axis
-    varies slowest, and the index along the last quickest.  The opposite holds
-    for Fortran-, or column-major, mode.
+    In C-like (row-major) order, in two dimensions, the row index varies the
+    slowest, and the column index the quickest.  This can be generalized to
+    multiple dimensions, where row-major order implies that the index along the
+    first axis varies slowest, and the index along the last quickest.  The
+    opposite holds for Fortran-like, or column-major, index ordering.
 
     Examples
     --------
@@ -1056,9 +1369,8 @@
     array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11])
 
     """
-    if not hasattr(a, 'ravel'):
-        a = numpypy.array(a)
-    return a.ravel(order=order)
+    return asarray(a).ravel(order)
+
 
 def nonzero(a):
     """
@@ -1180,9 +1492,11 @@
     (2,)
 
     """
-    if not hasattr(a, 'shape'):
-        a = numpypy.array(a)
-    return a.shape
+    try:
+        result = a.shape
+    except AttributeError:
+        result = asarray(a).shape
+    return result
 
 
 def compress(condition, a, axis=None, out=None):
@@ -1217,7 +1531,8 @@
     See Also
     --------
     take, choose, diag, diagonal, select
-    ndarray.compress : Equivalent method.
+    ndarray.compress : Equivalent method in ndarray
+    np.extract: Equivalent method when working on 1-D arrays
     numpy.doc.ufuncs : Section "Output arguments"
 
     Examples
@@ -1244,7 +1559,11 @@
     array([2])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        compress = a.compress
+    except AttributeError:
+        return _wrapit(a, 'compress', condition, axis, out)
+    return compress(condition, axis, out)
 
 
 def clip(a, a_min, a_max, out=None):
@@ -1297,12 +1616,14 @@
     array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8])
 
     """
-    if not hasattr(a, 'clip'):
-        a = numpypy.array(a)
-    return a.clip(a_min, a_max, out=out)
-
-
-def sum(a, axis=None, dtype=None, out=None):
+    try:
+        clip = a.clip
+    except AttributeError:
+        return _wrapit(a, 'clip', a_min, a_max, out)
+    return clip(a_min, a_max, out)
+
+
+def sum(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Sum of array elements over a given axis.
 
@@ -1310,9 +1631,16 @@
     ----------
     a : array_like
         Elements to sum.
-    axis : integer, optional
-        Axis over which the sum is taken. By default `axis` is None,
-        and all elements are summed.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a sum is performed.
+        The default (`axis` = `None`) is perform a sum over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a sum is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     dtype : dtype, optional
         The type of the returned array and of the accumulator in which
         the elements are summed.  By default, the dtype of `a` is used.
@@ -1325,6 +1653,10 @@
         (the shape of `a` with `axis` removed, i.e.,
         ``numpy.delete(a.shape, axis)``).  Its type is preserved. See
         `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1368,13 +1700,25 @@
     -128
 
     """
-    assert dtype is None
-    if not hasattr(a, "sum"):
-        a = numpypy.array(a)
-    return a.sum(axis=axis, out=out)
-
-
-def product (a, axis=None, dtype=None, out=None):
+    if isinstance(a, _gentype):
+        res = _sum_(a)
+        if out is not None:
+            out[...] = res
+            return out
+        return res
+    elif type(a) is not mu.ndarray:
+        try:
+            sum = a.sum
+        except AttributeError:
+            return _methods._sum(a, axis=axis, dtype=dtype,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameters here...
+        return sum(axis=axis, dtype=dtype, out=out)
+    else:
+        return _methods._sum(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
+
+def product (a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Return the product of array elements over a given axis.
 
@@ -1383,10 +1727,10 @@
     prod : equivalent function; see for details.
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def sometrue(a, axis=None, out=None):
+    return um.multiply.reduce(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+
+def sometrue(a, axis=None, out=None, keepdims=False):
     """
     Check whether some values are true.
 
@@ -1397,14 +1741,14 @@
     any : equivalent function
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'any'):
-        a = numpypy.array(a)
-    return a.any()
-
-
-def alltrue (a, axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.any(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.any(axis=axis, out=out)
+
+def alltrue (a, axis=None, out=None, keepdims=False):
     """
     Check if all elements of input array are true.
 
@@ -1413,13 +1757,14 @@
     numpy.all : Equivalent function; see for details.
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'all'):
-        a = numpypy.array(a)
-    return a.all()
-
-def any(a,axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.all(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.all(axis=axis, out=out)
+
+def any(a, axis=None, out=None, keepdims=False):
     """
     Test whether any array element along a given axis evaluates to True.
 
@@ -1429,17 +1774,26 @@
     ----------
     a : array_like
         Input array or object that can be converted to an array.
-    axis : int, optional
-        Axis along which a logical OR is performed.  The default
-        (`axis` = `None`) is to perform a logical OR over a flattened
-        input array. `axis` may be negative, in which case it counts
-        from the last to the first axis.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical OR reduction is performed.
+        The default (`axis` = `None`) is perform a logical OR over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a reduction is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     out : ndarray, optional
         Alternate output array in which to place the result.  It must have
         the same shape as the expected output and its type is preserved
         (e.g., if it is of type float, then it will remain so, returning
         1.0 for True and 0.0 for False, regardless of the type of `a`).
         See `doc.ufuncs` (Section "Output arguments") for details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1483,14 +1837,14 @@
     (191614240, 191614240)
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'any'):
-        a = numpypy.array(a)
-    return a.any()
-
-
-def all(a,axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.any(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.any(axis=axis, out=out)
+
+def all(a, axis=None, out=None, keepdims=False):
     """
     Test whether all array elements along a given axis evaluate to True.
 
@@ -1498,17 +1852,26 @@
     ----------
     a : array_like
         Input array or object that can be converted to an array.
-    axis : int, optional
-        Axis along which a logical AND is performed.
-        The default (`axis` = `None`) is to perform a logical AND
-        over a flattened input array.  `axis` may be negative, in which
-        case it counts from the last to the first axis.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical AND reduction is performed.
+        The default (`axis` = `None`) is perform a logical OR over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a reduction is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     out : ndarray, optional
         Alternate output array in which to place the result.
         It must have the same shape as the expected output and its
         type is preserved (e.g., if ``dtype(out)`` is float, the result
         will consist of 0.0's and 1.0's).  See `doc.ufuncs` (Section
         "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1547,12 +1910,12 @@
     (28293632, 28293632, array([ True], dtype=bool))
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'all'):
-        a = numpypy.array(a)
-    return a.all()
-
+    arr = asanyarray(a)
+
+    try:
+        return arr.all(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.all(axis=axis, out=out)
 
 def cumsum (a, axis=None, dtype=None, out=None):
     """
@@ -1592,6 +1955,8 @@
 
     trapz : Integration of array values using the composite trapezoidal rule.
 
+    diff :  Calculate the n-th order discrete difference along given axis.
+
     Notes
     -----
     Arithmetic is modular when using integer types, and no error is
@@ -1616,7 +1981,11 @@
            [ 4,  9, 15]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumsum = a.cumsum
+    except AttributeError:
+        return _wrapit(a, 'cumsum', axis, dtype, out)
+    return cumsum(axis, dtype, out)
 
 
 def cumproduct(a, axis=None, dtype=None, out=None):
@@ -1629,7 +1998,11 @@
     cumprod : equivalent function; see for details.
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumprod = a.cumprod
+    except AttributeError:
+        return _wrapit(a, 'cumprod', axis, dtype, out)
+    return cumprod(axis, dtype, out)
 
 
 def ptp(a, axis=None, out=None):
@@ -1670,10 +2043,14 @@
     array([1, 1])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def amax(a, axis=None, out=None):
+    try:
+        ptp = a.ptp
+    except AttributeError:
+        return _wrapit(a, 'ptp', axis, out)
+    return ptp(axis, out)
+
+
+def amax(a, axis=None, out=None, keepdims=False):
     """
     Return the maximum of an array or maximum along an axis.
 
@@ -1682,11 +2059,15 @@
     a : array_like
         Input data.
     axis : int, optional
-        Axis along which to operate.  By default flattened input is used.
+        Axis along which to operate.  By default, flattened input is used.
     out : ndarray, optional
-        Alternate output array in which to place the result.  Must be of
-        the same shape and buffer length as the expected output.  See
-        `doc.ufuncs` (Section "Output arguments") for more details.
+        Alternative output array in which to place the result.  Must
+        be of the same shape and buffer length as the expected output.
+        See `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1697,27 +2078,40 @@
 
     See Also
     --------
-    nanmax : NaN values are ignored instead of being propagated.
-    fmax : same behavior as the C99 fmax function.
-    argmax : indices of the maximum values.
+    amin :
+        The minimum value of an array along a given axis, propagating any NaNs.
+    nanmax :
+        The maximum value of an array along a given axis, ignoring any NaNs.
+    maximum :
+        Element-wise maximum of two arrays, propagating any NaNs.
+    fmax :
+        Element-wise maximum of two arrays, ignoring any NaNs.
+    argmax :
+        Return the indices of the maximum values.
+
+    nanmin, minimum, fmin
 
     Notes
     -----
     NaN values are propagated, that is if at least one item is NaN, the
-    corresponding max value will be NaN as well.  To ignore NaN values
+    corresponding max value will be NaN as well. To ignore NaN values
     (MATLAB behavior), please use nanmax.
 
+    Don't use `amax` for element-wise comparison of 2 arrays; when
+    ``a.shape[0]`` is 2, ``maximum(a[0], a[1])`` is faster than
+    ``amax(a, axis=0)``.
+
     Examples
     --------
     >>> a = np.arange(4).reshape((2,2))
     >>> a
     array([[0, 1],
            [2, 3]])
-    >>> np.amax(a)
+    >>> np.amax(a)           # Maximum of the flattened array
     3
-    >>> np.amax(a, axis=0)
+    >>> np.amax(a, axis=0)   # Maxima along the first axis
     array([2, 3])
-    >>> np.amax(a, axis=1)
+    >>> np.amax(a, axis=1)   # Maxima along the second axis
     array([1, 3])
 
     >>> b = np.arange(5, dtype=np.float)
@@ -1728,14 +2122,19 @@
     4.0
 
     """
-    if not hasattr(a, "max"):
-        a = numpypy.array(a)
-    if a.size < 1:
-        return numpypy.array([])
-    return a.max(axis=axis, out=out)
-
-
-def amin(a, axis=None, out=None):
+    if type(a) is not mu.ndarray:
+        try:
+            amax = a.max
+        except AttributeError:
+            return _methods._amax(a, axis=axis,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameter
+        return amax(axis=axis, out=out)
+    else:
+        return _methods._amax(a, axis=axis,
+                            out=out, keepdims=keepdims)
+
+def amin(a, axis=None, out=None, keepdims=False):
     """
     Return the minimum of an array or minimum along an axis.
 
@@ -1744,30 +2143,47 @@
     a : array_like
         Input data.
     axis : int, optional
-        Axis along which to operate.  By default a flattened input is used.
+        Axis along which to operate.  By default, flattened input is used.
     out : ndarray, optional
         Alternative output array in which to place the result.  Must
         be of the same shape and buffer length as the expected output.
         See `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
-    amin : ndarray
-        A new array or a scalar array with the result.
+    amin : ndarray or scalar
+        Minimum of `a`. If `axis` is None, the result is a scalar value.
+        If `axis` is given, the result is an array of dimension
+        ``a.ndim - 1``.
 
     See Also
     --------
-    nanmin: nan values are ignored instead of being propagated
-    fmin: same behavior as the C99 fmin function
-    argmin: Return the indices of the minimum values.
-
-    amax, nanmax, fmax
+    amax :
+        The maximum value of an array along a given axis, propagating any NaNs.
+    nanmin :
+        The minimum value of an array along a given axis, ignoring any NaNs.
+    minimum :
+        Element-wise minimum of two arrays, propagating any NaNs.
+    fmin :
+        Element-wise minimum of two arrays, ignoring any NaNs.
+    argmin :
+        Return the indices of the minimum values.
+
+    nanmax, maximum, fmax
 
     Notes
     -----
-    NaN values are propagated, that is if at least one item is nan, the
-    corresponding min value will be nan as well. To ignore NaN values (matlab
-    behavior), please use nanmin.
+    NaN values are propagated, that is if at least one item is NaN, the
+    corresponding min value will be NaN as well. To ignore NaN values
+    (MATLAB behavior), please use nanmin.
+
+    Don't use `amin` for element-wise comparison of 2 arrays; when
+    ``a.shape[0]`` is 2, ``minimum(a[0], a[1])`` is faster than
+    ``amin(a, axis=0)``.
 
     Examples
     --------
@@ -1777,9 +2193,9 @@
            [2, 3]])
     >>> np.amin(a)           # Minimum of the flattened array
     0
-    >>> np.amin(a, axis=0)         # Minima along the first axis
+    >>> np.amin(a, axis=0)   # Minima along the first axis
     array([0, 1])
-    >>> np.amin(a, axis=1)         # Minima along the second axis
+    >>> np.amin(a, axis=1)   # Minima along the second axis
     array([0, 2])
 
     >>> b = np.arange(5, dtype=np.float)
@@ -1790,11 +2206,17 @@
     0.0
 
     """
-    if not hasattr(a, 'min'):
-        a = numpypy.array(a)
-    if a.size < 1:
-        return numpypy.array([])
-    return a.min(axis=axis, out=out)
+    if type(a) is not mu.ndarray:
+        try:
+            amin = a.min
+        except AttributeError:
+            return _methods._amin(a, axis=axis,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameter
+        return amin(axis=axis, out=out)
+    else:
+        return _methods._amin(a, axis=axis,
+                            out=out, keepdims=keepdims)
 
 def alen(a):
     """
@@ -1807,7 +2229,7 @@
 
     Returns
     -------
-    l : int
+    alen : int
        Length of the first dimension of `a`.
 
     See Also
@@ -1823,12 +2245,13 @@
     7
 
     """
-    if not hasattr(a, 'shape'):
-        a = numpypy.array(a)
-    return a.shape[0]
-
-
-def prod(a, axis=None, dtype=None, out=None):
+    try:
+        return len(a)
+    except TypeError:
+        return len(array(a, ndmin=1))
+
+
+def prod(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Return the product of array elements over a given axis.
 
@@ -1836,9 +2259,16 @@
     ----------
     a : array_like
         Input data.
-    axis : int, optional
-        Axis over which the product is taken.  By default, the product
-        of all elements is calculated.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a product is performed.
+        The default (`axis` = `None`) is perform a product over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a product is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     dtype : data-type, optional
         The data-type of the returned array, as well as of the accumulator
         in which the elements are multiplied.  By default, if `a` is of
@@ -1849,6 +2279,10 @@
         Alternative output array in which to place the result. It must have
         the same shape as the expected output, but the type of the
         output values will be cast if necessary.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1902,8 +2336,16 @@
     True
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
+    if type(a) is not mu.ndarray:
+        try:
+            prod = a.prod
+        except AttributeError:
+            return _methods._prod(a, axis=axis, dtype=dtype,
+                                out=out, keepdims=keepdims)
+        return prod(axis=axis, dtype=dtype, out=out)
+    else:
+        return _methods._prod(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
 
 def cumprod(a, axis=None, dtype=None, out=None):
     """
@@ -1965,7 +2407,11 @@
            [  4,  20, 120]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumprod = a.cumprod
+    except AttributeError:
+        return _wrapit(a, 'cumprod', axis, dtype, out)
+    return cumprod(axis, dtype, out)
 
 
 def ndim(a):
@@ -1999,9 +2445,10 @@
     0
 
     """
-    if not hasattr(a, 'ndim'):
-        a = numpypy.array(a)
-    return a.ndim
+    try:
+        return a.ndim
+    except AttributeError:
+        return asarray(a).ndim
 
 
 def rank(a):
@@ -2044,9 +2491,10 @@
     0
 
     """
-    if not hasattr(a, 'ndim'):
-        a = numpypy.array(a)
-    return a.ndim
+    try:
+        return a.ndim
+    except AttributeError:
+        return asarray(a).ndim
 
 
 def size(a, axis=None):
@@ -2083,7 +2531,16 @@
     2
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    if axis is None:
+        try:
+            return a.size
+        except AttributeError:
+            return asarray(a).size
+    else:
+        try:
+            return a.shape[axis]
+        except AttributeError:
+            return asarray(a).shape[axis]
 
 
 def around(a, decimals=0, out=None):
@@ -2152,7 +2609,11 @@
     array([ 0,  0,  0, 10])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        round = a.round
+    except AttributeError:
+        return _wrapit(a, 'round', decimals, out)
+    return round(decimals, out)
 
 
 def round_(a, decimals=0, out=None):
@@ -2166,10 +2627,14 @@
     around : equivalent function
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def mean(a, axis=None, dtype=None, out=None):
+    try:
+        round = a.round
+    except AttributeError:
+        return _wrapit(a, 'round', decimals, out)
+    return round(decimals, out)
+
+
+def mean(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Compute the arithmetic mean along the specified axis.
 
@@ -2194,6 +2659,10 @@
         is ``None``; if provided, it must have the same shape as the
         expected output, but the type will be cast if necessary.
         See `doc.ufuncs` for details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2204,6 +2673,7 @@
     See Also
     --------
     average : Weighted average
+    std, var, nanmean, nanstd, nanvar
 
     Notes
     -----
@@ -2240,14 +2710,17 @@
     0.55000000074505806
 
     """
-    assert dtype is None
-    assert out is None
-    if not hasattr(a, "mean"):
-        a = numpypy.array(a)
-    return a.mean(axis=axis)
-
-
-def std(a, axis=None, dtype=None, out=None, ddof=0):
+    if type(a) is not mu.ndarray:
+        try:
+            mean = a.mean
+            return mean(axis=axis, dtype=dtype, out=out)
+        except AttributeError:
+            pass
+
+    return _methods._mean(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
+
+def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
     """
     Compute the standard deviation along the specified axis.
 
@@ -2274,6 +2747,10 @@
         Means Delta Degrees of Freedom.  The divisor used in calculations
         is ``N - ddof``, where ``N`` represents the number of elements.
         By default `ddof` is zero.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2283,7 +2760,7 @@
 
     See Also
     --------
-    var, mean
+    var, mean, nanmean, nanstd, nanvar
     numpy.doc.ufuncs : Section "Output arguments"
 
     Notes
@@ -2291,14 +2768,15 @@
     The standard deviation is the square root of the average of the squared
     deviations from the mean, i.e., ``std = sqrt(mean(abs(x - x.mean())**2))``.
 
-    The average squared deviation is normally calculated as ``x.sum() / N``, where
-    ``N = len(x)``.  If, however, `ddof` is specified, the divisor ``N - ddof``
-    is used instead. In standard statistical practice, ``ddof=1`` provides an
-    unbiased estimator of the variance of the infinite population. ``ddof=0``
-    provides a maximum likelihood estimate of the variance for normally
-    distributed variables. The standard deviation computed in this function
-    is the square root of the estimated variance, so even with ``ddof=1``, it
-    will not be an unbiased estimate of the standard deviation per se.
+    The average squared deviation is normally calculated as
+    ``x.sum() / N``, where ``N = len(x)``.  If, however, `ddof` is specified,
+    the divisor ``N - ddof`` is used instead. In standard statistical
+    practice, ``ddof=1`` provides an unbiased estimator of the variance
+    of the infinite population. ``ddof=0`` provides a maximum likelihood
+    estimate of the variance for normally distributed variables. The
+    standard deviation computed in this function is the square root of
+    the estimated variance, so even with ``ddof=1``, it will not be an
+    unbiased estimate of the standard deviation per se.
 
     Note that, for complex numbers, `std` takes the absolute
     value before squaring, so that the result is always real and nonnegative.
@@ -2333,15 +2811,18 @@
     0.44999999925552653
 
     """
-    assert dtype is None
-    assert out is None
-    assert ddof == 0
-    if not hasattr(a, "std"):
-        a = numpypy.array(a)
-    return a.std(axis=axis)
-
-
-def var(a, axis=None, dtype=None, out=None, ddof=0):
+    if type(a) is not mu.ndarray:
+        try:
+            std = a.std
+            return std(axis=axis, dtype=dtype, out=out, ddof=ddof)
+        except AttributeError:
+            pass
+
+    return _methods._std(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
+                                keepdims=keepdims)
+
+def var(a, axis=None, dtype=None, out=None, ddof=0,
+                            keepdims=False):
     """
     Compute the variance along the specified axis.
 
@@ -2369,6 +2850,10 @@
         "Delta Degrees of Freedom": the divisor used in the calculation is
         ``N - ddof``, where ``N`` represents the number of elements. By
         default `ddof` is zero.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2378,8 +2863,7 @@
 
     See Also
     --------
-    std : Standard deviation
-    mean : Average
+    std , mean, nanmean, nanstd, nanvar
     numpy.doc.ufuncs : Section "Output arguments"
 
     Notes
@@ -2408,9 +2892,9 @@
     >>> a = np.array([[1,2],[3,4]])
     >>> np.var(a)
     1.25
-    >>> np.var(a,0)
+    >>> np.var(a, axis=0)
     array([ 1.,  1.])
-    >>> np.var(a,1)
+    >>> np.var(a, axis=1)
     array([ 0.25,  0.25])
 
     In single precision, var() can be inaccurate:
@@ -2421,7 +2905,7 @@
     >>> np.var(a)
     0.20405951142311096
 
-    Computing the standard deviation in float64 is more accurate:
+    Computing the variance in float64 is more accurate:
 
     >>> np.var(a, dtype=np.float64)
     0.20249999932997387
@@ -2429,9 +2913,12 @@
     0.20250000000000001
 
     """
-    assert dtype is None
-    assert out is None
-    assert ddof == 0
-    if not hasattr(a, "var"):
-        a = numpypy.array(a)
-    return a.var(axis=axis)
+    if type(a) is not mu.ndarray:
+        try:
+            var = a.var
+            return var(axis=axis, dtype=dtype, out=out, ddof=ddof)
+        except AttributeError:
+            pass
+
+    return _methods._var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
+                                keepdims=keepdims)
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -1,21 +1,24 @@
+from __future__ import division, absolute_import, print_function
+
 __all__ = [
-           'newaxis', 'ufunc',
+           'newaxis', 'ufunc', 'argwhere',
            'asarray', 'asanyarray', 'base_repr',
            'array_repr', 'array_str', 'set_string_function',
-           'array_equal', 'outer', 'vdot', 'identity', 'little_endian',
-           'Inf', 'inf', 'infty', 'Infinity', 'nan', 'NaN', 'False_', 'True_',
-           'seterr',
+           'array_equal', 'array_equiv', 'outer', 'vdot', 'identity', 'little_endian',
+           'seterr', 'flatnonzero',
+           'Inf', 'inf', 'infty', 'Infinity',
+           'nan', 'NaN', 'False_', 'True_',
           ]
 
 import sys
-import multiarray
-from multiarray import *
+from . import multiarray
+from .multiarray import *
 del set_string_function
 del typeinfo
-import umath
-from umath import *
-import numerictypes
-from numerictypes import *
+from . import umath
+from .umath import *
+from . import numerictypes
+from .numerictypes import *
 
 def extend_all(module):
     adict = {}
@@ -41,6 +44,76 @@
 def seterr(**args):
     return args
 
+def asarray(a, dtype=None, order=None):
+    """
+    Convert the input to an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data, in any form that can be converted to an array.  This
+        includes lists, lists of tuples, tuples, tuples of tuples, tuples
+        of lists and ndarrays.
+    dtype : data-type, optional
+        By default, the data-type is inferred from the input data.
+    order : {'C', 'F'}, optional
+        Whether to use row-major ('C') or column-major ('F' for FORTRAN)
+        memory representation.  Defaults to 'C'.
+
+    Returns
+    -------
+    out : ndarray
+        Array interpretation of `a`.  No copy is performed if the input
+        is already an ndarray.  If `a` is a subclass of ndarray, a base
+        class ndarray is returned.
+
+    See Also
+    --------
+    asanyarray : Similar function which passes through subclasses.
+    ascontiguousarray : Convert input to a contiguous array.
+    asfarray : Convert input to a floating point ndarray.
+    asfortranarray : Convert input to an ndarray with column-major
+                     memory order.
+    asarray_chkfinite : Similar function which checks input for NaNs and Infs.
+    fromiter : Create an array from an iterator.
+    fromfunction : Construct an array by executing a function on grid
+                   positions.
+
+    Examples
+    --------
+    Convert a list into an array:
+
+    >>> a = [1, 2]
+    >>> np.asarray(a)
+    array([1, 2])
+
+    Existing arrays are not copied:
+
+    >>> a = np.array([1, 2])
+    >>> np.asarray(a) is a
+    True
+
+    If `dtype` is set, array is copied only if dtype does not match:
+
+    >>> a = np.array([1, 2], dtype=np.float32)
+    >>> np.asarray(a, dtype=np.float32) is a
+    True
+    >>> np.asarray(a, dtype=np.float64) is a
+    False
+
+    Contrary to `asanyarray`, ndarray subclasses are not passed through:
+
+    >>> issubclass(np.matrix, np.ndarray)
+    True
+    >>> a = np.matrix([[1, 2]])
+    >>> np.asarray(a) is a
+    False
+    >>> np.asanyarray(a) is a
+    True
+
+    """
+    return array(a, dtype, copy=False, order=order)
+
 def asanyarray(a, dtype=None, order=None):
     """
     Convert the input to an ndarray, but pass ndarray subclasses through.
@@ -93,6 +166,85 @@
     """
     return array(a, dtype, copy=False, order=order, subok=True)
 
+def argwhere(a):
+    """
+    Find the indices of array elements that are non-zero, grouped by element.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+
+    Returns
+    -------
+    index_array : ndarray
+        Indices of elements that are non-zero. Indices are grouped by element.
+
+    See Also
+    --------
+    where, nonzero
+
+    Notes
+    -----
+    ``np.argwhere(a)`` is the same as ``np.transpose(np.nonzero(a))``.
+
+    The output of ``argwhere`` is not suitable for indexing arrays.
+    For this purpose use ``where(a)`` instead.
+
+    Examples
+    --------
+    >>> x = np.arange(6).reshape(2,3)
+    >>> x
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.argwhere(x>1)
+    array([[0, 2],
+           [1, 0],
+           [1, 1],
+           [1, 2]])
+
+    """
+    return transpose(asanyarray(a).nonzero())
+
+def flatnonzero(a):
+    """
+    Return indices that are non-zero in the flattened version of a.
+
+    This is equivalent to a.ravel().nonzero()[0].
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array.
+
+    Returns
+    -------
+    res : ndarray
+        Output array, containing the indices of the elements of `a.ravel()`
+        that are non-zero.
+
+    See Also
+    --------
+    nonzero : Return the indices of the non-zero elements of the input array.
+    ravel : Return a 1-D array containing the elements of the input array.
+
+    Examples
+    --------
+    >>> x = np.arange(-2, 3)
+    >>> x
+    array([-2, -1,  0,  1,  2])
+    >>> np.flatnonzero(x)
+    array([0, 1, 3, 4])
+
+    Use the indices of the non-zero elements as an index array to extract
+    these elements:
+
+    >>> x.ravel()[np.flatnonzero(x)]
+    array([-2, -1,  1,  2])
+
+    """
+    return a.ravel().nonzero()[0]
+
 def base_repr(number, base=2, padding=0):
     """
     Return a string representation of a number in the given base system.
@@ -148,7 +300,7 @@
 
 
 #Use numarray's printing function
-from arrayprint import array2string
+from .arrayprint import array2string
 
 _typelessdata = [int_, float_]#, complex_]
 # XXX
@@ -381,75 +533,49 @@
         return False
     return bool((a1 == a2).all())
 
-def asarray(a, dtype=None, order=None):
+def array_equiv(a1, a2):
     """
-    Convert the input to an array.
+    Returns True if input arrays are shape consistent and all elements equal.
+
+    Shape consistent means they are either the same shape, or one input array
+    can be broadcasted to create the same shape as the other one.
 
     Parameters
     ----------
-    a : array_like
-        Input data, in any form that can be converted to an array.  This
-        includes lists, lists of tuples, tuples, tuples of tuples, tuples
-        of lists and ndarrays.
-    dtype : data-type, optional
-        By default, the data-type is inferred from the input data.
-    order : {'C', 'F'}, optional
-        Whether to use row-major ('C') or column-major ('F' for FORTRAN)
-        memory representation.  Defaults to 'C'.
+    a1, a2 : array_like
+        Input arrays.
 
     Returns
     -------
-    out : ndarray
-        Array interpretation of `a`.  No copy is performed if the input
-        is already an ndarray.  If `a` is a subclass of ndarray, a base
-        class ndarray is returned.
-
-    See Also
-    --------
-    asanyarray : Similar function which passes through subclasses.
-    ascontiguousarray : Convert input to a contiguous array.
-    asfarray : Convert input to a floating point ndarray.
-    asfortranarray : Convert input to an ndarray with column-major
-                     memory order.
-    asarray_chkfinite : Similar function which checks input for NaNs and Infs.
-    fromiter : Create an array from an iterator.
-    fromfunction : Construct an array by executing a function on grid
-                   positions.
+    out : bool
+        True if equivalent, False otherwise.
 
     Examples
     --------
-    Convert a list into an array:
-
-    >>> a = [1, 2]
-    >>> np.asarray(a)
-    array([1, 2])
-
-    Existing arrays are not copied:
-
-    >>> a = np.array([1, 2])
-    >>> np.asarray(a) is a
+    >>> np.array_equiv([1, 2], [1, 2])
     True
-
-    If `dtype` is set, array is copied only if dtype does not match:
-
-    >>> a = np.array([1, 2], dtype=np.float32)
-    >>> np.asarray(a, dtype=np.float32) is a
-    True
-    >>> np.asarray(a, dtype=np.float64) is a
+    >>> np.array_equiv([1, 2], [1, 3])
     False
 
-    Contrary to `asanyarray`, ndarray subclasses are not passed through:
+    Showing the shape equivalence:
 
-    >>> issubclass(np.matrix, np.ndarray)
+    >>> np.array_equiv([1, 2], [[1, 2], [1, 2]])
     True
-    >>> a = np.matrix([[1, 2]])

From noreply at buildbot.pypy.org  Mon Oct 21 18:07:03 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 21 Oct 2013 18:07:03 +0200 (CEST)
Subject: [pypy-commit] pypy fast_cffi_list_init: close to-be-merged branch
Message-ID: <20131021160703.55E8B1C0149@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fast_cffi_list_init
Changeset: r67482:bcad65dfb0d0
Date: 2013-10-21 18:03 +0200
http://bitbucket.org/pypy/pypy/changeset/bcad65dfb0d0/

Log:	close to-be-merged branch


From noreply at buildbot.pypy.org  Mon Oct 21 18:07:05 2013
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 21 Oct 2013 18:07:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge the fast_cffi_list_init branch.
Message-ID: <20131021160705.11DFC1C0149@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r67483:147153568452
Date: 2013-10-21 18:06 +0200
http://bitbucket.org/pypy/pypy/changeset/147153568452/

Log:	merge the fast_cffi_list_init branch.

	This adds special support for converting a list with IntStrategy to
	a cffi long[] array and viceversa, and the same for FloatStrategy
	and double[] array. Such conversions are now done by doing a simple
	memcpy from/to the storage of the lists, and can be exploited by
	serialization libraries such as msgpack to provide a super-fast
	(de)serialization of lists

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -239,6 +239,18 @@
         # _____ this code is here to support testing only _____
         return self
 
+    def unpackiterable_int(self, space):
+        lst = space.listview_int(self)
+        if lst:
+            return lst[:]
+        return None
+
+    def unpackiterable_float(self, space):
+        lst = space.listview_float(self)
+        if lst:
+            return lst[:]
+        return None
+
 
 class W_InterpIterable(W_Root):
     def __init__(self, space, w_iterable):
@@ -838,6 +850,22 @@
         return self._unpackiterable_known_length_jitlook(w_iterator,
                                                          expected_length)
 
+
+    def unpackiterable_int(self, w_obj):
+        """
+        Return a RPython list of unwrapped ints out of w_obj. The list is
+        guaranteed to be acopy of the actual data contained in w_obj, so you
+        can freely modify it. It might return None if not supported.
+        """
+        return w_obj.unpackiterable_int(self)
+
+    def unpackiterable_float(self, w_obj):
+        """
+        Same as unpackiterable_int, but for floats.
+        """
+        return w_obj.unpackiterable_float(self)
+
+
     def length_hint(self, w_obj, default):
         """Return the length of an object, consulting its __length_hint__
         method if necessary.
@@ -895,6 +923,20 @@
         """
         return None
 
+    def listview_int(self, w_list):
+        """ Return a list of unwrapped int out of a list of int. If the
+        argument is not a list or does not contain only int, return None.
+        May return None anyway.
+        """
+        return None
+
+    def listview_float(self, w_list):
+        """ Return a list of unwrapped float out of a list of float. If the
+        argument is not a list or does not contain only float, return None.
+        May return None anyway.
+        """
+        return None
+
     def view_as_kwargs(self, w_dict):
         """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
         strings and one of wrapped values. otherwise return (None, None)
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -282,6 +282,12 @@
     def iter(self):
         return self.ctype.iter(self)
 
+    def unpackiterable_int(self, space):
+        return self.ctype.aslist_int(self)
+
+    def unpackiterable_float(self, space):
+        return self.ctype.aslist_float(self)
+
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
         misc.write_raw_signed_data(self._cdata, source, self.ctype.size)
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -105,6 +105,26 @@
     def iter(self, cdata):
         return W_CDataIter(self.space, self.ctitem, cdata)
 
+    def aslist_int(self, cdata):
+        from rpython.rlib.rarray import populate_list_from_raw_array
+        if self.ctitem.is_long():
+            res = []
+            buf = rffi.cast(rffi.LONGP, cdata._cdata)
+            length = cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        return None
+
+    def aslist_float(self, cdata):
+        from rpython.rlib.rarray import populate_list_from_raw_array
+        if self.ctitem.is_double():
+            res = []
+            buf = rffi.cast(rffi.DOUBLEP, cdata._cdata)
+            length = cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        return None
+
     def get_vararg_type(self):
         return self.ctptr
 
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -43,6 +43,12 @@
     def is_unichar_ptr_or_array(self):
         return False
 
+    def is_long(self):
+        return False
+
+    def is_double(self):
+        return False
+
     def newp(self, w_init):
         space = self.space
         raise operationerrfmt(space.w_TypeError,
@@ -163,6 +169,9 @@
                               "cdata '%s' does not support iteration",
                               self.name)
 
+    def unpackiterable_int(self, cdata):
+        return None
+
     def get_vararg_type(self):
         return self
 
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -85,7 +85,6 @@
             return self.space.wrap(s)
         return W_CType.string(self, cdataobj, maxlen)
 
-
 class W_CTypePrimitiveCharOrUniChar(W_CTypePrimitive):
     _attrs_ = []
     is_primitive_integer = True
@@ -171,6 +170,9 @@
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
 
+    def is_long(self):
+        return self.size == rffi.sizeof(lltype.Signed)
+
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
 
@@ -274,6 +276,9 @@
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
 
+    def is_double(self):
+        return self.size == rffi.sizeof(lltype.Float)
+
     def cast(self, w_ob):
         space = self.space
         if isinstance(w_ob, cdataobj.W_CData):
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -42,6 +42,12 @@
     def is_char_or_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveCharOrUniChar)
 
+    def aslist_int(self, cdata):
+        return None
+
+    def aslist_float(self, cdata):
+        return None
+
     def cast(self, w_ob):
         # cast to a pointer, to a funcptr, or to an array.
         # Note that casting to an array is an extension to the C language,
@@ -58,19 +64,45 @@
             value = rffi.cast(rffi.CCHARP, value)
         return cdataobj.W_CData(space, value, self)
 
+    def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
+        from rpython.rlib.rarray import copy_list_to_raw_array
+        int_list = self.space.listview_int(w_ob)
+        float_list = self.space.listview_float(w_ob)
+        #
+        if self.ctitem.is_long() and int_list is not None:
+            cdata = rffi.cast(rffi.LONGP, cdata)
+            copy_list_to_raw_array(int_list, cdata)
+            return True
+        #
+        if self.ctitem.is_double() and float_list is not None:
+            cdata = rffi.cast(rffi.DOUBLEP, cdata)
+            copy_list_to_raw_array(float_list, cdata)
+            return True
+        #
+        return False
+
+    def _convert_array_from_listview(self, cdata, w_ob):
+        space = self.space
+        lst_w = space.listview(w_ob)
+        if self.length >= 0 and len(lst_w) > self.length:
+            raise operationerrfmt(space.w_IndexError,
+                "too many initializers for '%s' (got %d)",
+                                  self.name, len(lst_w))
+        ctitem = self.ctitem
+        for i in range(len(lst_w)):
+            ctitem.convert_from_object(cdata, lst_w[i])
+            cdata = rffi.ptradd(cdata, ctitem.size)
+
     def convert_array_from_object(self, cdata, w_ob):
         space = self.space
+        if self._convert_array_from_list_strategy_maybe(cdata, w_ob):
+            # the fast path worked, we are done now
+            return
+        #
+        # continue with the slow path
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
-            lst_w = space.listview(w_ob)
-            if self.length >= 0 and len(lst_w) > self.length:
-                raise operationerrfmt(space.w_IndexError,
-                    "too many initializers for '%s' (got %d)",
-                                      self.name, len(lst_w))
-            ctitem = self.ctitem
-            for i in range(len(lst_w)):
-                ctitem.convert_from_object(cdata, lst_w[i])
-                cdata = rffi.ptradd(cdata, ctitem.size)
+            self._convert_array_from_listview(cdata, w_ob)
         elif (self.can_cast_anything or
               (self.ctitem.is_primitive_integer and
                self.ctitem.size == rffi.sizeof(lltype.Char))):
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -0,0 +1,100 @@
+# side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule()
+from pypy.module._cffi_backend import misc
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+
+class AppTest_fast_path_from_list(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def setup_method(self, meth):
+        def forbidden(self, *args):
+            assert False, 'The slow path is forbidden'
+        self._original = W_CTypePtrOrArray._convert_array_from_listview.im_func
+        W_CTypePtrOrArray._convert_array_from_listview = forbidden
+
+    def teardown_method(self, meth):
+        W_CTypePtrOrArray._convert_array_from_listview = self._original
+
+    def test_fast_init_from_list(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, None)
+        buf = _cffi_backend.newp(LONG_ARRAY, [1, 2, 3])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == 3
+
+    def test_fast_init_from_list_float(self):
+        import _cffi_backend
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        P_DOUBLE = _cffi_backend.new_pointer_type(DOUBLE)
+        DOUBLE_ARRAY = _cffi_backend.new_array_type(P_DOUBLE, None)
+        buf = _cffi_backend.newp(DOUBLE_ARRAY, [1.1, 2.2, 3.3])
+        assert buf[0] == 1.1
+        assert buf[1] == 2.2
+        assert buf[2] == 3.3
+
+
+class AppTest_fast_path_to_list(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def setup_method(self, meth):
+        from pypy.interpreter import gateway
+        from rpython.rlib import rarray
+        #
+        self.count = 0
+        def get_count(*args):
+            return self.space.wrap(self.count)
+        self.w_get_count = self.space.wrap(gateway.interp2app(get_count))
+        #
+        original = rarray.populate_list_from_raw_array
+        def populate_list_from_raw_array(*args):
+            self.count += 1
+            return original(*args)
+        self._original = original
+        rarray.populate_list_from_raw_array = populate_list_from_raw_array
+        #
+        self.w_runappdirect = self.space.wrap(self.runappdirect)
+
+
+    def teardown_method(self, meth):
+        from rpython.rlib import rarray
+        rarray.populate_list_from_raw_array = self._original
+
+    def test_list_int(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, 3)
+        buf = _cffi_backend.newp(LONG_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_TypeError_if_no_length(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, 3)
+        buf = _cffi_backend.newp(LONG_ARRAY)
+        pbuf = _cffi_backend.cast(P_LONG, buf)
+        raises(TypeError, "list(pbuf)")
+
+
+    def test_list_float(self):
+        import _cffi_backend
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        P_DOUBLE = _cffi_backend.new_pointer_type(DOUBLE)
+        DOUBLE_ARRAY = _cffi_backend.new_array_type(P_DOUBLE, 3)
+        buf = _cffi_backend.newp(DOUBLE_ARRAY)
+        buf[0] = 1.1
+        buf[1] = 2.2
+        buf[2] = 3.3
+        lst = list(buf)
+        assert lst == [1.1, 2.2, 3.3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -139,6 +139,8 @@
 
 class W_ListObject(W_Root):
 
+    strategy = None
+
     def __init__(self, space, wrappeditems, sizehint=-1):
         assert isinstance(wrappeditems, list)
         self.space = space
@@ -290,6 +292,11 @@
         """Return the items in the list as unwrapped ints. If the list does not
         use the list strategy, return None."""
         return self.strategy.getitems_int(self)
+
+    def getitems_float(self):
+        """Return the items in the list as unwrapped floats. If the list does not
+        use the list strategy, return None."""
+        return self.strategy.getitems_float(self)
     # ___________________________________________________
 
     def mul(self, times):
@@ -755,6 +762,9 @@
     def getitems_int(self, w_list):
         return None
 
+    def getitems_float(self, w_list):
+        return None
+
     def getstorage_copy(self, w_list):
         raise NotImplementedError
 
@@ -939,11 +949,16 @@
             w_list.__init__(space, w_iterable.getitems_copy())
             return
 
-        intlist = space.listview_int(w_iterable)
+        intlist = space.unpackiterable_int(w_iterable)
         if intlist is not None:
             w_list.strategy = strategy = space.fromcache(IntegerListStrategy)
-            # need to copy because intlist can share with w_iterable
-            w_list.lstorage = strategy.erase(intlist[:])
+            w_list.lstorage = strategy.erase(intlist)
+            return
+
+        floatlist = space.unpackiterable_float(w_iterable)
+        if floatlist is not None:
+            w_list.strategy = strategy = space.fromcache(FloatListStrategy)
+            w_list.lstorage = strategy.erase(floatlist)
             return
 
         strlist = space.listview_str(w_iterable)
@@ -1573,6 +1588,9 @@
         if reverse:
             l.reverse()
 
+    def getitems_float(self, w_list):
+        return self.unerase(w_list.lstorage)
+
 
 class StringListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -472,6 +472,15 @@
             return w_obj.getitems_int()
         return None
 
+    def listview_float(self, w_obj):
+        if type(w_obj) is W_ListObject:
+            return w_obj.getitems_float()
+        # dict and set don't have FloatStrategy, so we can just ignore them
+        # for now
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_float()
+        return None
+
     def view_as_kwargs(self, w_dict):
         if type(w_dict) is W_DictMultiObject:
             return w_dict.view_as_kwargs()
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -645,6 +645,20 @@
         w_l = W_ListObject(space, [space.wrap(1), space.wrap(2), space.wrap(3)])
         assert self.space.listview_int(w_l) == [1, 2, 3]
 
+    def test_listview_float_list(self):
+        space = self.space
+        w_l = W_ListObject(space, [space.wrap(1.1), space.wrap(2.2), space.wrap(3.3)])
+        assert self.space.listview_float(w_l) == [1.1, 2.2, 3.3]
+
+    def test_unpackiterable_int_list(self):
+        space = self.space
+        w_l = W_ListObject(space, [space.wrap(1), space.wrap(2), space.wrap(3)])
+        list_orig = self.space.listview_int(w_l)
+        list_copy = self.space.unpackiterable_int(w_l)
+        assert list_orig == list_copy == [1, 2, 3]
+        list_copy[0] = 42
+        assert list_orig == [1, 2, 3]
+
 
 class TestW_ListStrategiesDisabled:
     spaceconfig = {"objspace.std.withliststrategies": False}
diff --git a/rpython/rlib/rarray.py b/rpython/rlib/rarray.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rarray.py
@@ -0,0 +1,75 @@
+from rpython.annotator import model as annmodel
+from rpython.annotator.listdef import ListDef
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib import jit
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.tool.pairtype import pair
+
+def copy_list_to_raw_array(lst, array):
+    for i, item in enumerate(lst):
+        array[i] = item
+
+def populate_list_from_raw_array(lst, array, length):
+    lst[:] = [array[i] for i in range(length)]
+
+
+
+class Entry(ExtRegistryEntry):
+    _about_ = copy_list_to_raw_array
+
+    def compute_result_annotation(self, *s_args):
+        pass
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        v_list, v_buf = hop.inputargs(*hop.args_r)
+        return hop.gendirectcall(ll_copy_list_to_raw_array, v_list, v_buf)
+
+
+class Entry(ExtRegistryEntry):
+    _about_ = populate_list_from_raw_array
+
+    def compute_result_annotation(self, s_list, s_array, s_length):
+        s_item = annmodel.lltype_to_annotation(s_array.ll_ptrtype.TO.OF)
+        s_newlist = self.bookkeeper.newlist(s_item)
+        s_newlist.listdef.resize()
+        pair(s_list, s_newlist).union()
+
+    def specialize_call(self, hop):
+        v_list, v_buf, v_length = hop.inputargs(*hop.args_r)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_populate_list_from_raw_array, v_list, v_buf, v_length)
+
+
+ at specialize.ll()
+def get_raw_buf(ptr):
+    ofs = llmemory.itemoffsetof(lltype.typeOf(ptr).TO, 0)
+    return llmemory.cast_ptr_to_adr(ptr) + ofs
+get_raw_buf._always_inline_ = True
+
+
+ at jit.dont_look_inside
+def ll_copy_list_to_raw_array(ll_list, dst_ptr):
+    # this code is delicate: we must ensure that there are no GC operations
+    # around the call to raw_memcopy
+    #
+    ITEM = lltype.typeOf(dst_ptr).TO.OF
+    size = llmemory.sizeof(ITEM) * ll_list.ll_length()
+    # start of no-GC section
+    src_adr = get_raw_buf(ll_list.ll_items())
+    dst_adr = get_raw_buf(dst_ptr)
+    llmemory.raw_memcopy(src_adr, dst_adr, size)
+    # end of no-GC section
+
+
+ at jit.dont_look_inside
+def ll_populate_list_from_raw_array(ll_list, src_ptr, length):
+    ITEM = lltype.typeOf(src_ptr).TO.OF
+    size = llmemory.sizeof(ITEM) * length
+    ll_list._ll_resize(length)
+    # start of no-GC section
+    src_adr = get_raw_buf(src_ptr)
+    dst_adr = get_raw_buf(ll_list.ll_items())
+    llmemory.raw_memcopy(src_adr, dst_adr, size)
+    # end of no-GC section
diff --git a/rpython/rlib/test/test_rarray.py b/rpython/rlib/test/test_rarray.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rarray.py
@@ -0,0 +1,64 @@
+from rpython.rlib.rarray import copy_list_to_raw_array, populate_list_from_raw_array
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.test.tool import BaseRtypingTest
+
+
+
+class TestRArray(BaseRtypingTest):
+
+    def test_copy_list_to_raw_array(self):
+        ARRAY = rffi.CArray(lltype.Signed)
+        buf = lltype.malloc(ARRAY, 4, flavor='raw')
+        lst = [1, 2, 3, 4]
+        copy_list_to_raw_array(lst, buf)
+        for i in range(4):
+            assert buf[i] == i+1
+        lltype.free(buf, flavor='raw')
+        
+
+    def test_copy_list_to_raw_array_rtyped(self):
+        INTARRAY = rffi.CArray(lltype.Signed)
+        FLOATARRAY = rffi.CArray(lltype.Float)
+        def fn():
+            buf = lltype.malloc(INTARRAY, 3, flavor='raw')
+            lst = [1, 2, 3]
+            copy_list_to_raw_array(lst, buf)
+            for i in range(3):
+                assert buf[i] == lst[i]
+            #
+            buf2 = lltype.malloc(FLOATARRAY, 3, flavor='raw')
+            lst = [1.1, 2.2, 3.3]
+            copy_list_to_raw_array(lst, buf2)
+            for i in range(3):
+                assert buf2[i] == lst[i]
+            #
+            lltype.free(buf, flavor='raw')
+            lltype.free(buf2, flavor='raw')
+        self.interpret(fn, [])
+
+    def test_new_list_from_raw_array(self):
+        INTARRAY = rffi.CArray(lltype.Signed)
+        buf = lltype.malloc(INTARRAY, 4, flavor='raw')
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        buf[3] = 4
+        lst = []
+        populate_list_from_raw_array(lst, buf, 4)
+        assert lst == [1, 2, 3, 4]
+        lltype.free(buf, flavor='raw')
+
+    def test_new_list_from_raw_array_rtyped(self):
+        INTARRAY = rffi.CArray(lltype.Signed)
+        def fn():
+            buf = lltype.malloc(INTARRAY, 4, flavor='raw')
+            buf[0] = 1
+            buf[1] = 2
+            buf[2] = 3
+            buf[3] = 4
+            lst = []
+            populate_list_from_raw_array(lst, buf, 4)
+            assert lst == [1, 2, 3, 4]
+            lltype.free(buf, flavor='raw')
+        #
+        self.interpret(fn, [])

From noreply at buildbot.pypy.org  Tue Oct 22 00:36:33 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue, 22 Oct 2013 00:36:33 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20131021223633.524C71C01F5@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67484:c17b5f220446
Date: 2013-10-15 12:47 -0700
http://bitbucket.org/pypy/pypy/changeset/c17b5f220446/

Log:	merge default

diff too long, truncating to 2000 out of 8503 lines

diff --git a/lib_pypy/numpy.py b/lib_pypy/numpy.py
deleted file mode 100644
--- a/lib_pypy/numpy.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import warnings
-import sys
-if 'numpypy' not in sys.modules:
-    warnings.warn(
-        "The 'numpy' module of PyPy is in-development and not complete. "
-        "To avoid this warning, write 'import numpypy as numpy'. ",
-        UserWarning) # XXX is this the best warning type?
-
-from numpypy import *
-import numpypy
-__all__ = numpypy.__all__
-del numpypy
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,51 +1,17 @@
-import core
-from core import *
-import lib
-from lib import *
+from . import core
+from .core import *
+from . import lib
+from .lib import *
 
 from __builtin__ import bool, int, long, float, complex, object, unicode, str
-from core import abs, max, min
+
+from .core import round, abs, max, min
 
 __version__ = '1.7.0'
 
-import os
-def get_include():
-    """
-    Return the directory that contains the NumPy \\*.h header files.
-
-    Extension modules that need to compile against NumPy should use this
-    function to locate the appropriate include directory.
-
-    Notes
-    -----
-    When using ``distutils``, for example in ``setup.py``.
-    ::
-
-        import numpy as np
-        ...
-        Extension('extension_name', ...
-                include_dirs=[np.get_include()])
-        ...
-
-    """
-    import numpy
-    if getattr(numpy, 'show_config', None) is None:
-        # running from numpy source directory
-        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-        return os.path.join(head, '../include')
-    else:
-        # using installed numpy core headers
-        import numpy.core as core
-        d = os.path.join(os.path.dirname(core.__file__), 'include')
-    return d
-
-
-
-__all__ = ['__version__', 'get_include']
+__all__ = ['__version__']
 __all__ += core.__all__
 __all__ += lib.__all__
 
 #import sys
 #sys.modules.setdefault('numpy', sys.modules['numpypy'])
-
-
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
--- a/lib_pypy/numpypy/core/__init__.py
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -1,12 +1,17 @@
-import numeric
-from numeric import *
-import fromnumeric
-from fromnumeric import *
-import shape_base
-from shape_base import *
+from __future__ import division, absolute_import, print_function
 
-from fromnumeric import amax as max, amin as min
-from numeric import absolute as abs
+from . import multiarray
+from . import umath
+from . import numeric
+from .numeric import *
+from . import fromnumeric
+from .fromnumeric import *
+from . import shape_base
+from .shape_base import *
+
+from .fromnumeric import amax as max, amin as min, \
+    round_ as round
+from .numeric import absolute as abs
 
 __all__ = []
 __all__ += numeric.__all__
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -1,36 +1,53 @@
-######################################################################    
+######################################################################
 # This is a copy of numpy/core/fromnumeric.py modified for numpypy
 ######################################################################
-# Each name in __all__ was a function in  'numeric' that is now 
-# a method in 'numpy'.
-# When the corresponding method is added to numpypy BaseArray
-# each function should be added as a module function 
-# at the applevel 
-# This can be as simple as doing the following
-#
-# def func(a, ...):
-#     if not hasattr(a, 'func')
-#         a = numpypy.array(a)
-#     return a.func(...)
-#
-######################################################################
-
-import numpypy
-import _numpypy
-
-# Module containing non-deprecated functions borrowed from Numeric.
-__docformat__ = "restructuredtext en"
+"""Module containing non-deprecated functions borrowed from Numeric.
+
+"""
+from __future__ import division, absolute_import, print_function
+
+import types
+
+from . import multiarray as mu
+from . import umath as um
+from . import numerictypes as nt
+from .numeric import asarray, array, asanyarray, concatenate
+from . import _methods
+
+
+# functions that are methods
+__all__ = [
+        'alen', 'all', 'alltrue', 'amax', 'amin', 'any', 'argmax',
+        'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip',
+        'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean',
+        'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put',
+        'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_',
+        'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze',
+        'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var',
+        ]
+
+
+try:
+    _gentype = types.GeneratorType
+except AttributeError:
+    _gentype = type(None)
+
+# save away Python sum
+_sum_ = sum
 
 # functions that are now methods
-__all__ = ['take', 'reshape', 'choose', 'repeat', 'put',
-           'swapaxes', 'transpose', 'sort', 'argsort', 'argmax', 'argmin',
-           'searchsorted', 'alen',
-           'resize', 'diagonal', 'trace', 'ravel', 'nonzero', 'shape',
-           'compress', 'clip', 'sum', 'product', 'prod', 'sometrue', 'alltrue',
-           'any', 'all', 'cumsum', 'cumproduct', 'cumprod', 'ptp', 'ndim',
-           'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze',
-           'amax', 'amin',
-          ]
+def _wrapit(obj, method, *args, **kwds):
+    try:
+        wrap = obj.__array_wrap__
+    except AttributeError:
+        wrap = None
+    result = getattr(asarray(obj), method)(*args, **kwds)
+    if wrap:
+        if not isinstance(result, mu.ndarray):
+            result = asarray(result)
+        result = wrap(result)
+    return result
+
 
 def take(a, indices, axis=None, out=None, mode='raise'):
     """
@@ -46,6 +63,10 @@
         The source array.
     indices : array_like
         The indices of the values to extract.
+
+        .. versionadded:: 1.8.0
+
+        Also allow scalars for indices.
     axis : int, optional
         The axis over which to select values. By default, the flattened
         input array is used.
@@ -85,8 +106,17 @@
     >>> a[indices]
     array([4, 3, 6])
 
+    If `indices` is not one dimensional, the output also has these dimensions.
+
+    >>> np.take(a, [[0, 1], [2, 3]])
+    array([[4, 3],
+           [5, 7]])
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        take = a.take
+    except AttributeError:
+        return _wrapit(a, 'take', indices, axis, out, mode)
+    return take(indices, axis, out, mode)
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -104,16 +134,23 @@
         One shape dimension can be -1. In this case, the value is inferred
         from the length of the array and remaining dimensions.
     order : {'C', 'F', 'A'}, optional
-        Determines whether the array data should be viewed as in C
-        (row-major) order, FORTRAN (column-major) order, or the C/FORTRAN
-        order should be preserved.
+        Read the elements of `a` using this index order, and place the elements
+        into the reshaped array using this index order.  'C' means to
+        read / write the elements using C-like index order, with the last axis index
+        changing fastest, back to the first axis index changing slowest.  'F'
+        means to read / write the elements using Fortran-like index order, with
+        the first index changing fastest, and the last index changing slowest.
+        Note that the 'C' and 'F' options take no account of the memory layout
+        of the underlying array, and only refer to the order of indexing.  'A'
+        means to read / write the elements in Fortran-like index order if `a` is
+        Fortran *contiguous* in memory, C-like order otherwise.
 
     Returns
     -------
     reshaped_array : ndarray
         This will be a new view object if possible; otherwise, it will
-        be a copy.
-
+        be a copy.  Note there is no guarantee of the *memory layout* (C- or
+        Fortran- contiguous) of the returned array.
 
     See Also
     --------
@@ -121,7 +158,6 @@
 
     Notes
     -----
-
     It is not always possible to change the shape of an array without
     copying the data. If you want an error to be raise if the data is copied,
     you should assign the new shape to the shape attribute of the array::
@@ -129,12 +165,39 @@
      >>> a = np.zeros((10, 2))
      # A transpose make the array non-contiguous
      >>> b = a.T
-     # Taking a view makes it possible to modify the shape without modiying the
+     # Taking a view makes it possible to modify the shape without modifying the
      # initial object.
      >>> c = b.view()
      >>> c.shape = (20)
      AttributeError: incompatible shape for a non-contiguous array
 
+    The `order` keyword gives the index ordering both for *fetching* the values
+    from `a`, and then *placing* the values into the output array.  For example,
+    let's say you have an array:
+
+    >>> a = np.arange(6).reshape((3, 2))
+    >>> a
+    array([[0, 1],
+           [2, 3],
+           [4, 5]])
+
+    You can think of reshaping as first raveling the array (using the given
+    index order), then inserting the elements from the raveled array into the
+    new array using the same kind of index ordering as was used for the
+    raveling.
+
+    >>> np.reshape(a, (2, 3)) # C-like index ordering
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(a, (2, 3), order='F') # Fortran-like index ordering
+    array([[0, 4, 3],
+           [2, 1, 5]])
+    >>> np.reshape(np.ravel(a, order='F'), (2, 3), order='F')
+    array([[0, 4, 3],
+           [2, 1, 5]])
 
     Examples
     --------
@@ -148,12 +211,13 @@
     array([[1, 2],
            [3, 4],
            [5, 6]])
-
     """
     assert order == 'C'
-    if not hasattr(a, 'reshape'):
-       a = numpypy.array(a)
-    return a.reshape(newshape)
+    try:
+        reshape = a.reshape
+    except AttributeError:
+        return _wrapit(a, 'reshape', newshape)
+    return reshape(newshape)
 
 
 def choose(a, choices, out=None, mode='raise'):
@@ -275,7 +339,11 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    return _numpypy.choose(a, choices, out, mode)
+    try:
+        choose = a.choose
+    except AttributeError:
+        return _wrapit(a, 'choose', choices, out=out, mode=mode)
+    return choose(choices, out=out, mode=mode)
 
 
 def repeat(a, repeats, axis=None):
@@ -317,7 +385,11 @@
            [3, 4]])
 
     """
-    return _numpypy.repeat(a, repeats, axis)
+    try:
+        repeat = a.repeat
+    except AttributeError:
+        return _wrapit(a, 'repeat', repeats, axis)
+    return repeat(repeats, axis)
 
 
 def put(a, ind, v, mode='raise'):
@@ -368,7 +440,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return a.put(ind, v, mode)
 
 
 def swapaxes(a, axis1, axis2):
@@ -412,7 +484,10 @@
             [3, 7]]])
 
     """
-    swapaxes = a.swapaxes
+    try:
+        swapaxes = a.swapaxes
+    except AttributeError:
+        return _wrapit(a, 'swapaxes', axis1, axis2)
     return swapaxes(axis1, axis2)
 
 
@@ -456,9 +531,158 @@
     """
     if axes is not None:
         raise NotImplementedError('No "axes" arg yet.')
-    if not hasattr(a, 'T'):
-        a = numpypy.array(a)
-    return a.T
+    try:
+        transpose = a.transpose
+    except AttributeError:
+        return _wrapit(a, 'transpose')
+    return transpose()
+
+
+def partition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Return a partitioned copy of an array.
+
+    Creates a copy of the array with its elements rearranged in such a way that
+    the value of the element in kth position is in the position it would be in
+    a sorted array. All elements smaller than the kth element are moved before
+    this element and all equal or greater are moved behind it. The ordering of
+    the elements in the two partitions is undefined.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to be sorted.
+    kth : int or sequence of ints
+        Element index to partition by. The kth value of the element will be in
+        its final sorted position and all smaller elements will be moved before
+        it and all equal or greater elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all elements
+        indexed by kth  of them into their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort. If None, the array is flattened before
+        sorting. The default is -1, which sorts along the last axis.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'.
+    order : list, optional
+        When `a` is a structured array, this argument specifies which fields
+        to compare first, second, and so on.  This list does not need to
+        include all of the fields.
+
+    Returns
+    -------
+    partitioned_array : ndarray
+        Array of the same type and shape as `a`.
+
+    See Also
+    --------
+    ndarray.partition : Method to sort an array in-place.
+    argpartition : Indirect partition.
+    sort : Full sorting
+
+    Notes
+    -----
+    The various selection algorithms are characterized by their average speed,
+    worst case performance, work space size, and whether they are stable. A
+    stable sort keeps items with the same key in the same relative order. The
+    three available algorithms have the following properties:
+
+    ================= ======= ============= ============ =======
+       kind            speed   worst case    work space  stable
+    ================= ======= ============= ============ =======
+    'introselect'        1        O(n)           0         no
+    ================= ======= ============= ============ =======
+
+    All the partition algorithms make temporary copies of the data when
+    partitioning along any but the last axis.  Consequently, partitioning
+    along the last axis is faster and uses less space than partitioning
+    along any other axis.
+
+    The sort order for complex numbers is lexicographic. If both the real
+    and imaginary parts are non-nan then the order is determined by the
+    real parts except when they are equal, in which case the order is
+    determined by the imaginary parts.
+
+    Examples
+    --------
+    >>> a = np.array([3, 4, 2, 1])
+    >>> np.partition(a, 3)
+    array([2, 1, 3, 4])
+
+    >>> np.partition(a, (1, 3))
+    array([1, 2, 3, 4])
+
+    """
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.partition(kth, axis=axis, kind=kind, order=order)
+    return a
+
+
+def argpartition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Perform an indirect partition along the given axis using the algorithm
+    specified by the `kind` keyword. It returns an array of indices of the
+    same shape as `a` that index data along the given axis in partitioned
+    order.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to sort.
+    kth : int or sequence of ints
+        Element index to partition by. The kth element will be in its final
+        sorted position and all smaller elements will be moved before it and
+        all larger elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all of them into
+        their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort.  The default is -1 (the last axis). If None,
+        the flattened array is used.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'
+    order : list, optional
+        When `a` is an array with fields defined, this argument specifies
+        which fields to compare first, second, etc.  Not all fields need be
+        specified.
+
+    Returns
+    -------
+    index_array : ndarray, int
+        Array of indices that partition `a` along the specified axis.
+        In other words, ``a[index_array]`` yields a sorted `a`.
+
+    See Also
+    --------
+    partition : Describes partition algorithms used.
+    ndarray.partition : Inplace partition.
+    argsort : Full indirect sort
+
+    Notes
+    -----
+    See `partition` for notes on the different selection algorithms.
+
+    Examples
+    --------
+    One dimensional array:
+
+    >>> x = np.array([3, 4, 2, 1])
+    >>> x[np.argpartition(x, 3)]
+    array([2, 1, 3, 4])
+    >>> x[np.argpartition(x, (1, 3))]
+    array([1, 2, 3, 4])
+
+    """
+    return a.argpartition(kth, axis, kind=kind, order=order)
+
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -489,6 +713,7 @@
     argsort : Indirect sort.
     lexsort : Indirect stable sort on multiple keys.
     searchsorted : Find elements in a sorted array.
+    partition : Partial sort.
 
     Notes
     -----
@@ -559,7 +784,13 @@
           dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.sort(axis, kind, order)
+    return a
 
 
 def argsort(a, axis=-1, kind='quicksort', order=None):
@@ -595,6 +826,7 @@
     sort : Describes sorting algorithms used.
     lexsort : Indirect stable sort with multiple keys.
     ndarray.sort : Inplace sort.
+    argpartition : Indirect partial sort.
 
     Notes
     -----
@@ -640,7 +872,11 @@
     array([0, 1])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        argsort = a.argsort
+    except AttributeError:
+        return _wrapit(a, 'argsort', axis, kind, order)
+    return argsort(axis, kind, order)
 
 
 def argmax(a, axis=None):
@@ -694,9 +930,11 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmax'):
-        a = numpypy.array(a)
-    return a.argmax()
+    try:
+        argmax = a.argmax
+    except AttributeError:
+        return _wrapit(a, 'argmax')
+    return argmax()
 
 
 def argmin(a, axis=None):
@@ -710,29 +948,37 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmin'):
-        a = numpypy.array(a)
-    return a.argmin()
-
-
-def searchsorted(a, v, side='left'):
+    try:
+        argmin = a.argmin
+    except AttributeError:
+        return _wrapit(a, 'argmin')
+    return argmin()
+
+
+def searchsorted(a, v, side='left', sorter=None):
     """
     Find indices where elements should be inserted to maintain order.
 
-    Find the indices into a sorted array `a` such that, if the corresponding
-    elements in `v` were inserted before the indices, the order of `a` would
-    be preserved.
+    Find the indices into a sorted array `a` such that, if the
+    corresponding elements in `v` were inserted before the indices, the
+    order of `a` would be preserved.
 
     Parameters
     ----------
     a : 1-D array_like
-        Input array, sorted in ascending order.
+        Input array. If `sorter` is None, then it must be sorted in
+        ascending order, otherwise `sorter` must be an array of indices
+        that sort it.
     v : array_like
         Values to insert into `a`.
     side : {'left', 'right'}, optional
-        If 'left', the index of the first suitable location found is given.  If
-        'right', return the last such index.  If there is no suitable
+        If 'left', the index of the first suitable location found is given.
+        If 'right', return the last such index.  If there is no suitable
         index, return either 0 or N (where N is the length of `a`).
+    sorter : 1-D array_like, optional
+        .. versionadded:: 1.7.0
+        Optional array of integer indices that sort array a into ascending
+        order. They are typically the result of argsort.
 
     Returns
     -------
@@ -761,7 +1007,11 @@
     array([0, 5, 1, 2])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        searchsorted = a.searchsorted
+    except AttributeError:
+        return _wrapit(a, 'searchsorted', v, side, sorter)
+    return searchsorted(v, side, sorter)
 
 
 def resize(a, new_shape):
@@ -802,10 +1052,30 @@
            [0, 1, 2, 3]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def squeeze(a):
+    if isinstance(new_shape, (int, nt.integer)):
+        new_shape = (new_shape,)
+    a = ravel(a)
+    Na = len(a)
+    if not Na: return mu.zeros(new_shape, a.dtype.char)
+    total_size = um.multiply.reduce(new_shape)
+    n_copies = int(total_size / Na)
+    extra = total_size % Na
+
+    if total_size == 0:
+        return a[:0]
+
+    if extra != 0:
+        n_copies = n_copies+1
+        extra = Na-extra
+
+    a = concatenate( (a,)*n_copies)
+    if extra > 0:
+        a = a[:-extra]
+
+    return reshape(a, new_shape)
+
+
+def squeeze(a, axis=None):
     """
     Remove single-dimensional entries from the shape of an array.
 
@@ -813,12 +1083,19 @@
     ----------
     a : array_like
         Input data.
+    axis : None or int or tuple of ints, optional
+        .. versionadded:: 1.7.0
+
+        Selects a subset of the single-dimensional entries in the
+        shape. If an axis is selected with shape entry greater than
+        one, an error is raised.
 
     Returns
     -------
     squeezed : ndarray
-        The input array, but with with all dimensions of length 1
-        removed.  Whenever possible, a view on `a` is returned.
+        The input array, but with with all or a subset of the
+        dimensions of length 1 removed. This is always `a` itself
+        or a view into `a`.
 
     Examples
     --------
@@ -827,9 +1104,20 @@
     (1, 3, 1)
     >>> np.squeeze(x).shape
     (3,)
+    >>> np.squeeze(x, axis=(2,)).shape
+    (1, 3)
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        squeeze = a.squeeze
+    except AttributeError:
+        return _wrapit(a, 'squeeze')
+    try:
+        # First try to use the new axis= parameter
+        return squeeze(axis=axis)
+    except TypeError:
+        # For backwards compatibility
+        return squeeze()
 
 
 def diagonal(a, offset=0, axis1=0, axis2=1):
@@ -844,6 +1132,27 @@
     removing `axis1` and `axis2` and appending an index to the right equal
     to the size of the resulting diagonals.
 
+    In versions of NumPy prior to 1.7, this function always returned a new,
+    independent array containing a copy of the values in the diagonal.
+
+    In NumPy 1.7 and 1.8, it continues to return a copy of the diagonal,
+    but depending on this fact is deprecated. Writing to the resulting
+    array continues to work as it used to, but a FutureWarning is issued.
+
+    In NumPy 1.9 it returns a read-only view on the original array.
+    Attempting to write to the resulting array will produce an error.
+
+    In NumPy 1.10, it will return a read/write view, Writing to the returned
+    array will alter your original array.
+
+    If you don't write to the array returned by this function, then you can
+    just ignore all of the above.
+
+    If you depend on the current behavior, then we suggest copying the
+    returned array explicitly, i.e., use ``np.diagonal(a).copy()`` instead of
+    just ``np.diagonal(a)``. This will work with both past and future versions
+    of NumPy.
+
     Parameters
     ----------
     a : array_like
@@ -913,7 +1222,7 @@
            [5, 7]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return asarray(a).diagonal(offset, axis1, axis2)
 
 
 def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None):
@@ -972,7 +1281,7 @@
     (2, 3)
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return asarray(a).trace(offset, axis1, axis2, dtype, out)
 
 def ravel(a, order='C'):
     """
@@ -984,21 +1293,25 @@
     Parameters
     ----------
     a : array_like
-        Input array.  The elements in ``a`` are read in the order specified by
+        Input array.  The elements in `a` are read in the order specified by
         `order`, and packed as a 1-D array.
     order : {'C','F', 'A', 'K'}, optional
-        The elements of ``a`` are read in this order. 'C' means to view
-        the elements in C (row-major) order. 'F' means to view the elements
-        in Fortran (column-major) order. 'A' means to view the elements
-        in 'F' order if a is Fortran contiguous, 'C' order otherwise.
-        'K' means to view the elements in the order they occur in memory,
-        except for reversing the data when strides are negative.
-        By default, 'C' order is used.
+        The elements of `a` are read using this index order. 'C' means to
+        index the elements in C-like order, with the last axis index changing
+        fastest, back to the first axis index changing slowest.   'F' means to
+        index the elements in Fortran-like index order, with the first index
+        changing fastest, and the last index changing slowest. Note that the 'C'
+        and 'F' options take no account of the memory layout of the underlying
+        array, and only refer to the order of axis indexing.  'A' means to read
+        the elements in Fortran-like index order if `a` is Fortran *contiguous*
+        in memory, C-like order otherwise.  'K' means to read the elements in
+        the order they occur in memory, except for reversing the data when
+        strides are negative.  By default, 'C' index order is used.
 
     Returns
     -------
     1d_array : ndarray
-        Output of the same dtype as `a`, and of shape ``(a.size(),)``.
+        Output of the same dtype as `a`, and of shape ``(a.size,)``.
 
     See Also
     --------
@@ -1008,11 +1321,11 @@
 
     Notes
     -----
-    In row-major order, the row index varies the slowest, and the column
-    index the quickest.  This can be generalized to multiple dimensions,
-    where row-major order implies that the index along the first axis
-    varies slowest, and the index along the last quickest.  The opposite holds
-    for Fortran-, or column-major, mode.
+    In C-like (row-major) order, in two dimensions, the row index varies the
+    slowest, and the column index the quickest.  This can be generalized to
+    multiple dimensions, where row-major order implies that the index along the
+    first axis varies slowest, and the index along the last quickest.  The
+    opposite holds for Fortran-like, or column-major, index ordering.
 
     Examples
     --------
@@ -1056,9 +1369,8 @@
     array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11])
 
     """
-    if not hasattr(a, 'ravel'):
-        a = numpypy.array(a)
-    return a.ravel(order=order)
+    return asarray(a).ravel(order)
+
 
 def nonzero(a):
     """
@@ -1180,9 +1492,11 @@
     (2,)
 
     """
-    if not hasattr(a, 'shape'):
-        a = numpypy.array(a)
-    return a.shape
+    try:
+        result = a.shape
+    except AttributeError:
+        result = asarray(a).shape
+    return result
 
 
 def compress(condition, a, axis=None, out=None):
@@ -1217,7 +1531,8 @@
     See Also
     --------
     take, choose, diag, diagonal, select
-    ndarray.compress : Equivalent method.
+    ndarray.compress : Equivalent method in ndarray
+    np.extract: Equivalent method when working on 1-D arrays
     numpy.doc.ufuncs : Section "Output arguments"
 
     Examples
@@ -1244,7 +1559,11 @@
     array([2])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        compress = a.compress
+    except AttributeError:
+        return _wrapit(a, 'compress', condition, axis, out)
+    return compress(condition, axis, out)
 
 
 def clip(a, a_min, a_max, out=None):
@@ -1297,12 +1616,14 @@
     array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8])
 
     """
-    if not hasattr(a, 'clip'):
-        a = numpypy.array(a)
-    return a.clip(a_min, a_max, out=out)
-
-
-def sum(a, axis=None, dtype=None, out=None):
+    try:
+        clip = a.clip
+    except AttributeError:
+        return _wrapit(a, 'clip', a_min, a_max, out)
+    return clip(a_min, a_max, out)
+
+
+def sum(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Sum of array elements over a given axis.
 
@@ -1310,9 +1631,16 @@
     ----------
     a : array_like
         Elements to sum.
-    axis : integer, optional
-        Axis over which the sum is taken. By default `axis` is None,
-        and all elements are summed.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a sum is performed.
+        The default (`axis` = `None`) is perform a sum over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a sum is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     dtype : dtype, optional
         The type of the returned array and of the accumulator in which
         the elements are summed.  By default, the dtype of `a` is used.
@@ -1325,6 +1653,10 @@
         (the shape of `a` with `axis` removed, i.e.,
         ``numpy.delete(a.shape, axis)``).  Its type is preserved. See
         `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1368,13 +1700,25 @@
     -128
 
     """
-    assert dtype is None
-    if not hasattr(a, "sum"):
-        a = numpypy.array(a)
-    return a.sum(axis=axis, out=out)
-
-
-def product (a, axis=None, dtype=None, out=None):
+    if isinstance(a, _gentype):
+        res = _sum_(a)
+        if out is not None:
+            out[...] = res
+            return out
+        return res
+    elif type(a) is not mu.ndarray:
+        try:
+            sum = a.sum
+        except AttributeError:
+            return _methods._sum(a, axis=axis, dtype=dtype,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameters here...
+        return sum(axis=axis, dtype=dtype, out=out)
+    else:
+        return _methods._sum(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
+
+def product (a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Return the product of array elements over a given axis.
 
@@ -1383,10 +1727,10 @@
     prod : equivalent function; see for details.
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def sometrue(a, axis=None, out=None):
+    return um.multiply.reduce(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+
+def sometrue(a, axis=None, out=None, keepdims=False):
     """
     Check whether some values are true.
 
@@ -1397,14 +1741,14 @@
     any : equivalent function
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'any'):
-        a = numpypy.array(a)
-    return a.any()
-
-
-def alltrue (a, axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.any(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.any(axis=axis, out=out)
+
+def alltrue (a, axis=None, out=None, keepdims=False):
     """
     Check if all elements of input array are true.
 
@@ -1413,13 +1757,14 @@
     numpy.all : Equivalent function; see for details.
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'all'):
-        a = numpypy.array(a)
-    return a.all()
-
-def any(a,axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.all(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.all(axis=axis, out=out)
+
+def any(a, axis=None, out=None, keepdims=False):
     """
     Test whether any array element along a given axis evaluates to True.
 
@@ -1429,17 +1774,26 @@
     ----------
     a : array_like
         Input array or object that can be converted to an array.
-    axis : int, optional
-        Axis along which a logical OR is performed.  The default
-        (`axis` = `None`) is to perform a logical OR over a flattened
-        input array. `axis` may be negative, in which case it counts
-        from the last to the first axis.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical OR reduction is performed.
+        The default (`axis` = `None`) is perform a logical OR over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a reduction is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     out : ndarray, optional
         Alternate output array in which to place the result.  It must have
         the same shape as the expected output and its type is preserved
         (e.g., if it is of type float, then it will remain so, returning
         1.0 for True and 0.0 for False, regardless of the type of `a`).
         See `doc.ufuncs` (Section "Output arguments") for details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1483,14 +1837,14 @@
     (191614240, 191614240)
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'any'):
-        a = numpypy.array(a)
-    return a.any()
-
-
-def all(a,axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.any(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.any(axis=axis, out=out)
+
+def all(a, axis=None, out=None, keepdims=False):
     """
     Test whether all array elements along a given axis evaluate to True.
 
@@ -1498,17 +1852,26 @@
     ----------
     a : array_like
         Input array or object that can be converted to an array.
-    axis : int, optional
-        Axis along which a logical AND is performed.
-        The default (`axis` = `None`) is to perform a logical AND
-        over a flattened input array.  `axis` may be negative, in which
-        case it counts from the last to the first axis.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical AND reduction is performed.
+        The default (`axis` = `None`) is perform a logical OR over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a reduction is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     out : ndarray, optional
         Alternate output array in which to place the result.
         It must have the same shape as the expected output and its
         type is preserved (e.g., if ``dtype(out)`` is float, the result
         will consist of 0.0's and 1.0's).  See `doc.ufuncs` (Section
         "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1547,12 +1910,12 @@
     (28293632, 28293632, array([ True], dtype=bool))
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'all'):
-        a = numpypy.array(a)
-    return a.all()
-
+    arr = asanyarray(a)
+
+    try:
+        return arr.all(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.all(axis=axis, out=out)
 
 def cumsum (a, axis=None, dtype=None, out=None):
     """
@@ -1592,6 +1955,8 @@
 
     trapz : Integration of array values using the composite trapezoidal rule.
 
+    diff :  Calculate the n-th order discrete difference along given axis.
+
     Notes
     -----
     Arithmetic is modular when using integer types, and no error is
@@ -1616,7 +1981,11 @@
            [ 4,  9, 15]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumsum = a.cumsum
+    except AttributeError:
+        return _wrapit(a, 'cumsum', axis, dtype, out)
+    return cumsum(axis, dtype, out)
 
 
 def cumproduct(a, axis=None, dtype=None, out=None):
@@ -1629,7 +1998,11 @@
     cumprod : equivalent function; see for details.
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumprod = a.cumprod
+    except AttributeError:
+        return _wrapit(a, 'cumprod', axis, dtype, out)
+    return cumprod(axis, dtype, out)
 
 
 def ptp(a, axis=None, out=None):
@@ -1670,10 +2043,14 @@
     array([1, 1])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def amax(a, axis=None, out=None):
+    try:
+        ptp = a.ptp
+    except AttributeError:
+        return _wrapit(a, 'ptp', axis, out)
+    return ptp(axis, out)
+
+
+def amax(a, axis=None, out=None, keepdims=False):
     """
     Return the maximum of an array or maximum along an axis.
 
@@ -1682,11 +2059,15 @@
     a : array_like
         Input data.
     axis : int, optional
-        Axis along which to operate.  By default flattened input is used.
+        Axis along which to operate.  By default, flattened input is used.
     out : ndarray, optional
-        Alternate output array in which to place the result.  Must be of
-        the same shape and buffer length as the expected output.  See
-        `doc.ufuncs` (Section "Output arguments") for more details.
+        Alternative output array in which to place the result.  Must
+        be of the same shape and buffer length as the expected output.
+        See `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1697,27 +2078,40 @@
 
     See Also
     --------
-    nanmax : NaN values are ignored instead of being propagated.
-    fmax : same behavior as the C99 fmax function.
-    argmax : indices of the maximum values.
+    amin :
+        The minimum value of an array along a given axis, propagating any NaNs.
+    nanmax :
+        The maximum value of an array along a given axis, ignoring any NaNs.
+    maximum :
+        Element-wise maximum of two arrays, propagating any NaNs.
+    fmax :
+        Element-wise maximum of two arrays, ignoring any NaNs.
+    argmax :
+        Return the indices of the maximum values.
+
+    nanmin, minimum, fmin
 
     Notes
     -----
     NaN values are propagated, that is if at least one item is NaN, the
-    corresponding max value will be NaN as well.  To ignore NaN values
+    corresponding max value will be NaN as well. To ignore NaN values
     (MATLAB behavior), please use nanmax.
 
+    Don't use `amax` for element-wise comparison of 2 arrays; when
+    ``a.shape[0]`` is 2, ``maximum(a[0], a[1])`` is faster than
+    ``amax(a, axis=0)``.
+
     Examples
     --------
     >>> a = np.arange(4).reshape((2,2))
     >>> a
     array([[0, 1],
            [2, 3]])
-    >>> np.amax(a)
+    >>> np.amax(a)           # Maximum of the flattened array
     3
-    >>> np.amax(a, axis=0)
+    >>> np.amax(a, axis=0)   # Maxima along the first axis
     array([2, 3])
-    >>> np.amax(a, axis=1)
+    >>> np.amax(a, axis=1)   # Maxima along the second axis
     array([1, 3])
 
     >>> b = np.arange(5, dtype=np.float)
@@ -1728,14 +2122,19 @@
     4.0
 
     """
-    if not hasattr(a, "max"):
-        a = numpypy.array(a)
-    if a.size < 1:
-        return numpypy.array([])
-    return a.max(axis=axis, out=out)
-
-
-def amin(a, axis=None, out=None):
+    if type(a) is not mu.ndarray:
+        try:
+            amax = a.max
+        except AttributeError:
+            return _methods._amax(a, axis=axis,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameter
+        return amax(axis=axis, out=out)
+    else:
+        return _methods._amax(a, axis=axis,
+                            out=out, keepdims=keepdims)
+
+def amin(a, axis=None, out=None, keepdims=False):
     """
     Return the minimum of an array or minimum along an axis.
 
@@ -1744,30 +2143,47 @@
     a : array_like
         Input data.
     axis : int, optional
-        Axis along which to operate.  By default a flattened input is used.
+        Axis along which to operate.  By default, flattened input is used.
     out : ndarray, optional
         Alternative output array in which to place the result.  Must
         be of the same shape and buffer length as the expected output.
         See `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
-    amin : ndarray
-        A new array or a scalar array with the result.
+    amin : ndarray or scalar
+        Minimum of `a`. If `axis` is None, the result is a scalar value.
+        If `axis` is given, the result is an array of dimension
+        ``a.ndim - 1``.
 
     See Also
     --------
-    nanmin: nan values are ignored instead of being propagated
-    fmin: same behavior as the C99 fmin function
-    argmin: Return the indices of the minimum values.
-
-    amax, nanmax, fmax
+    amax :
+        The maximum value of an array along a given axis, propagating any NaNs.
+    nanmin :
+        The minimum value of an array along a given axis, ignoring any NaNs.
+    minimum :
+        Element-wise minimum of two arrays, propagating any NaNs.
+    fmin :
+        Element-wise minimum of two arrays, ignoring any NaNs.
+    argmin :
+        Return the indices of the minimum values.
+
+    nanmax, maximum, fmax
 
     Notes
     -----
-    NaN values are propagated, that is if at least one item is nan, the
-    corresponding min value will be nan as well. To ignore NaN values (matlab
-    behavior), please use nanmin.
+    NaN values are propagated, that is if at least one item is NaN, the
+    corresponding min value will be NaN as well. To ignore NaN values
+    (MATLAB behavior), please use nanmin.
+
+    Don't use `amin` for element-wise comparison of 2 arrays; when
+    ``a.shape[0]`` is 2, ``minimum(a[0], a[1])`` is faster than
+    ``amin(a, axis=0)``.
 
     Examples
     --------
@@ -1777,9 +2193,9 @@
            [2, 3]])
     >>> np.amin(a)           # Minimum of the flattened array
     0
-    >>> np.amin(a, axis=0)         # Minima along the first axis
+    >>> np.amin(a, axis=0)   # Minima along the first axis
     array([0, 1])
-    >>> np.amin(a, axis=1)         # Minima along the second axis
+    >>> np.amin(a, axis=1)   # Minima along the second axis
     array([0, 2])
 
     >>> b = np.arange(5, dtype=np.float)
@@ -1790,11 +2206,17 @@
     0.0
 
     """
-    if not hasattr(a, 'min'):
-        a = numpypy.array(a)
-    if a.size < 1:
-        return numpypy.array([])
-    return a.min(axis=axis, out=out)
+    if type(a) is not mu.ndarray:
+        try:
+            amin = a.min
+        except AttributeError:
+            return _methods._amin(a, axis=axis,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameter
+        return amin(axis=axis, out=out)
+    else:
+        return _methods._amin(a, axis=axis,
+                            out=out, keepdims=keepdims)
 
 def alen(a):
     """
@@ -1807,7 +2229,7 @@
 
     Returns
     -------
-    l : int
+    alen : int
        Length of the first dimension of `a`.
 
     See Also
@@ -1823,12 +2245,13 @@
     7
 
     """
-    if not hasattr(a, 'shape'):
-        a = numpypy.array(a)
-    return a.shape[0]
-
-
-def prod(a, axis=None, dtype=None, out=None):
+    try:
+        return len(a)
+    except TypeError:
+        return len(array(a, ndmin=1))
+
+
+def prod(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Return the product of array elements over a given axis.
 
@@ -1836,9 +2259,16 @@
     ----------
     a : array_like
         Input data.
-    axis : int, optional
-        Axis over which the product is taken.  By default, the product
-        of all elements is calculated.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a product is performed.
+        The default (`axis` = `None`) is perform a product over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a product is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     dtype : data-type, optional
         The data-type of the returned array, as well as of the accumulator
         in which the elements are multiplied.  By default, if `a` is of
@@ -1849,6 +2279,10 @@
         Alternative output array in which to place the result. It must have
         the same shape as the expected output, but the type of the
         output values will be cast if necessary.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1902,8 +2336,16 @@
     True
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
+    if type(a) is not mu.ndarray:
+        try:
+            prod = a.prod
+        except AttributeError:
+            return _methods._prod(a, axis=axis, dtype=dtype,
+                                out=out, keepdims=keepdims)
+        return prod(axis=axis, dtype=dtype, out=out)
+    else:
+        return _methods._prod(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
 
 def cumprod(a, axis=None, dtype=None, out=None):
     """
@@ -1965,7 +2407,11 @@
            [  4,  20, 120]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumprod = a.cumprod
+    except AttributeError:
+        return _wrapit(a, 'cumprod', axis, dtype, out)
+    return cumprod(axis, dtype, out)
 
 
 def ndim(a):
@@ -1999,9 +2445,10 @@
     0
 
     """
-    if not hasattr(a, 'ndim'):
-        a = numpypy.array(a)
-    return a.ndim
+    try:
+        return a.ndim
+    except AttributeError:
+        return asarray(a).ndim
 
 
 def rank(a):
@@ -2044,9 +2491,10 @@
     0
 
     """
-    if not hasattr(a, 'ndim'):
-        a = numpypy.array(a)
-    return a.ndim
+    try:
+        return a.ndim
+    except AttributeError:
+        return asarray(a).ndim
 
 
 def size(a, axis=None):
@@ -2083,7 +2531,16 @@
     2
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    if axis is None:
+        try:
+            return a.size
+        except AttributeError:
+            return asarray(a).size
+    else:
+        try:
+            return a.shape[axis]
+        except AttributeError:
+            return asarray(a).shape[axis]
 
 
 def around(a, decimals=0, out=None):
@@ -2152,7 +2609,11 @@
     array([ 0,  0,  0, 10])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        round = a.round
+    except AttributeError:
+        return _wrapit(a, 'round', decimals, out)
+    return round(decimals, out)
 
 
 def round_(a, decimals=0, out=None):
@@ -2166,10 +2627,14 @@
     around : equivalent function
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def mean(a, axis=None, dtype=None, out=None):
+    try:
+        round = a.round
+    except AttributeError:
+        return _wrapit(a, 'round', decimals, out)
+    return round(decimals, out)
+
+
+def mean(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Compute the arithmetic mean along the specified axis.
 
@@ -2194,6 +2659,10 @@
         is ``None``; if provided, it must have the same shape as the
         expected output, but the type will be cast if necessary.
         See `doc.ufuncs` for details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2204,6 +2673,7 @@
     See Also
     --------
     average : Weighted average
+    std, var, nanmean, nanstd, nanvar
 
     Notes
     -----
@@ -2240,14 +2710,17 @@
     0.55000000074505806
 
     """
-    assert dtype is None
-    assert out is None
-    if not hasattr(a, "mean"):
-        a = numpypy.array(a)
-    return a.mean(axis=axis)
-
-
-def std(a, axis=None, dtype=None, out=None, ddof=0):
+    if type(a) is not mu.ndarray:
+        try:
+            mean = a.mean
+            return mean(axis=axis, dtype=dtype, out=out)
+        except AttributeError:
+            pass
+
+    return _methods._mean(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
+
+def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
     """
     Compute the standard deviation along the specified axis.
 
@@ -2274,6 +2747,10 @@
         Means Delta Degrees of Freedom.  The divisor used in calculations
         is ``N - ddof``, where ``N`` represents the number of elements.
         By default `ddof` is zero.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2283,7 +2760,7 @@
 
     See Also
     --------
-    var, mean
+    var, mean, nanmean, nanstd, nanvar
     numpy.doc.ufuncs : Section "Output arguments"
 
     Notes
@@ -2291,14 +2768,15 @@
     The standard deviation is the square root of the average of the squared
     deviations from the mean, i.e., ``std = sqrt(mean(abs(x - x.mean())**2))``.
 
-    The average squared deviation is normally calculated as ``x.sum() / N``, where
-    ``N = len(x)``.  If, however, `ddof` is specified, the divisor ``N - ddof``
-    is used instead. In standard statistical practice, ``ddof=1`` provides an
-    unbiased estimator of the variance of the infinite population. ``ddof=0``
-    provides a maximum likelihood estimate of the variance for normally
-    distributed variables. The standard deviation computed in this function
-    is the square root of the estimated variance, so even with ``ddof=1``, it
-    will not be an unbiased estimate of the standard deviation per se.
+    The average squared deviation is normally calculated as
+    ``x.sum() / N``, where ``N = len(x)``.  If, however, `ddof` is specified,
+    the divisor ``N - ddof`` is used instead. In standard statistical
+    practice, ``ddof=1`` provides an unbiased estimator of the variance
+    of the infinite population. ``ddof=0`` provides a maximum likelihood
+    estimate of the variance for normally distributed variables. The
+    standard deviation computed in this function is the square root of
+    the estimated variance, so even with ``ddof=1``, it will not be an
+    unbiased estimate of the standard deviation per se.
 
     Note that, for complex numbers, `std` takes the absolute
     value before squaring, so that the result is always real and nonnegative.
@@ -2333,15 +2811,18 @@
     0.44999999925552653
 
     """
-    assert dtype is None
-    assert out is None
-    assert ddof == 0
-    if not hasattr(a, "std"):
-        a = numpypy.array(a)
-    return a.std(axis=axis)
-
-
-def var(a, axis=None, dtype=None, out=None, ddof=0):
+    if type(a) is not mu.ndarray:
+        try:
+            std = a.std
+            return std(axis=axis, dtype=dtype, out=out, ddof=ddof)
+        except AttributeError:
+            pass
+
+    return _methods._std(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
+                                keepdims=keepdims)
+
+def var(a, axis=None, dtype=None, out=None, ddof=0,
+                            keepdims=False):
     """
     Compute the variance along the specified axis.
 
@@ -2369,6 +2850,10 @@
         "Delta Degrees of Freedom": the divisor used in the calculation is
         ``N - ddof``, where ``N`` represents the number of elements. By
         default `ddof` is zero.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2378,8 +2863,7 @@
 
     See Also
     --------
-    std : Standard deviation
-    mean : Average
+    std , mean, nanmean, nanstd, nanvar
     numpy.doc.ufuncs : Section "Output arguments"
 
     Notes
@@ -2408,9 +2892,9 @@
     >>> a = np.array([[1,2],[3,4]])
     >>> np.var(a)
     1.25
-    >>> np.var(a,0)
+    >>> np.var(a, axis=0)
     array([ 1.,  1.])
-    >>> np.var(a,1)
+    >>> np.var(a, axis=1)
     array([ 0.25,  0.25])
 
     In single precision, var() can be inaccurate:
@@ -2421,7 +2905,7 @@
     >>> np.var(a)
     0.20405951142311096
 
-    Computing the standard deviation in float64 is more accurate:
+    Computing the variance in float64 is more accurate:
 
     >>> np.var(a, dtype=np.float64)
     0.20249999932997387
@@ -2429,9 +2913,12 @@
     0.20250000000000001
 
     """
-    assert dtype is None
-    assert out is None
-    assert ddof == 0
-    if not hasattr(a, "var"):
-        a = numpypy.array(a)
-    return a.var(axis=axis)
+    if type(a) is not mu.ndarray:
+        try:
+            var = a.var
+            return var(axis=axis, dtype=dtype, out=out, ddof=ddof)
+        except AttributeError:
+            pass
+
+    return _methods._var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
+                                keepdims=keepdims)
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -1,21 +1,24 @@
+from __future__ import division, absolute_import, print_function
+
 __all__ = [
-           'newaxis', 'ufunc',
+           'newaxis', 'ufunc', 'argwhere',
            'asarray', 'asanyarray', 'base_repr',
            'array_repr', 'array_str', 'set_string_function',
-           'array_equal', 'outer', 'vdot', 'identity', 'little_endian',
-           'Inf', 'inf', 'infty', 'Infinity', 'nan', 'NaN', 'False_', 'True_',
-           'seterr',
+           'array_equal', 'array_equiv', 'outer', 'vdot', 'identity', 'little_endian',
+           'seterr', 'flatnonzero',
+           'Inf', 'inf', 'infty', 'Infinity',
+           'nan', 'NaN', 'False_', 'True_',
           ]
 
 import sys
-import multiarray
-from multiarray import *
+from . import multiarray
+from .multiarray import *
 del set_string_function
 del typeinfo
-import umath
-from umath import *
-import numerictypes
-from numerictypes import *
+from . import umath
+from .umath import *
+from . import numerictypes
+from .numerictypes import *
 
 def extend_all(module):
     adict = {}
@@ -41,6 +44,76 @@
 def seterr(**args):
     return args
 
+def asarray(a, dtype=None, order=None):
+    """
+    Convert the input to an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data, in any form that can be converted to an array.  This
+        includes lists, lists of tuples, tuples, tuples of tuples, tuples
+        of lists and ndarrays.
+    dtype : data-type, optional
+        By default, the data-type is inferred from the input data.
+    order : {'C', 'F'}, optional
+        Whether to use row-major ('C') or column-major ('F' for FORTRAN)
+        memory representation.  Defaults to 'C'.
+
+    Returns
+    -------
+    out : ndarray
+        Array interpretation of `a`.  No copy is performed if the input
+        is already an ndarray.  If `a` is a subclass of ndarray, a base
+        class ndarray is returned.
+
+    See Also
+    --------
+    asanyarray : Similar function which passes through subclasses.
+    ascontiguousarray : Convert input to a contiguous array.
+    asfarray : Convert input to a floating point ndarray.
+    asfortranarray : Convert input to an ndarray with column-major
+                     memory order.
+    asarray_chkfinite : Similar function which checks input for NaNs and Infs.
+    fromiter : Create an array from an iterator.
+    fromfunction : Construct an array by executing a function on grid
+                   positions.
+
+    Examples
+    --------
+    Convert a list into an array:
+
+    >>> a = [1, 2]
+    >>> np.asarray(a)
+    array([1, 2])
+
+    Existing arrays are not copied:
+
+    >>> a = np.array([1, 2])
+    >>> np.asarray(a) is a
+    True
+
+    If `dtype` is set, array is copied only if dtype does not match:
+
+    >>> a = np.array([1, 2], dtype=np.float32)
+    >>> np.asarray(a, dtype=np.float32) is a
+    True
+    >>> np.asarray(a, dtype=np.float64) is a
+    False
+
+    Contrary to `asanyarray`, ndarray subclasses are not passed through:
+
+    >>> issubclass(np.matrix, np.ndarray)
+    True
+    >>> a = np.matrix([[1, 2]])
+    >>> np.asarray(a) is a
+    False
+    >>> np.asanyarray(a) is a
+    True
+
+    """
+    return array(a, dtype, copy=False, order=order)
+
 def asanyarray(a, dtype=None, order=None):
     """
     Convert the input to an ndarray, but pass ndarray subclasses through.
@@ -93,6 +166,85 @@
     """
     return array(a, dtype, copy=False, order=order, subok=True)
 
+def argwhere(a):
+    """
+    Find the indices of array elements that are non-zero, grouped by element.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+
+    Returns
+    -------
+    index_array : ndarray
+        Indices of elements that are non-zero. Indices are grouped by element.
+
+    See Also
+    --------
+    where, nonzero
+
+    Notes
+    -----
+    ``np.argwhere(a)`` is the same as ``np.transpose(np.nonzero(a))``.
+
+    The output of ``argwhere`` is not suitable for indexing arrays.
+    For this purpose use ``where(a)`` instead.
+
+    Examples
+    --------
+    >>> x = np.arange(6).reshape(2,3)
+    >>> x
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.argwhere(x>1)
+    array([[0, 2],
+           [1, 0],
+           [1, 1],
+           [1, 2]])
+
+    """
+    return transpose(asanyarray(a).nonzero())
+
+def flatnonzero(a):
+    """
+    Return indices that are non-zero in the flattened version of a.
+
+    This is equivalent to a.ravel().nonzero()[0].
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array.
+
+    Returns
+    -------
+    res : ndarray
+        Output array, containing the indices of the elements of `a.ravel()`
+        that are non-zero.
+
+    See Also
+    --------
+    nonzero : Return the indices of the non-zero elements of the input array.
+    ravel : Return a 1-D array containing the elements of the input array.
+
+    Examples
+    --------
+    >>> x = np.arange(-2, 3)
+    >>> x
+    array([-2, -1,  0,  1,  2])
+    >>> np.flatnonzero(x)
+    array([0, 1, 3, 4])
+
+    Use the indices of the non-zero elements as an index array to extract
+    these elements:
+
+    >>> x.ravel()[np.flatnonzero(x)]
+    array([-2, -1,  1,  2])
+
+    """
+    return a.ravel().nonzero()[0]
+
 def base_repr(number, base=2, padding=0):
     """
     Return a string representation of a number in the given base system.
@@ -148,7 +300,7 @@
 
 
 #Use numarray's printing function
-from arrayprint import array2string
+from .arrayprint import array2string
 
 _typelessdata = [int_, float_]#, complex_]
 # XXX
@@ -381,75 +533,49 @@
         return False
     return bool((a1 == a2).all())
 
-def asarray(a, dtype=None, order=None):
+def array_equiv(a1, a2):
     """
-    Convert the input to an array.
+    Returns True if input arrays are shape consistent and all elements equal.
+
+    Shape consistent means they are either the same shape, or one input array
+    can be broadcasted to create the same shape as the other one.
 
     Parameters
     ----------
-    a : array_like
-        Input data, in any form that can be converted to an array.  This
-        includes lists, lists of tuples, tuples, tuples of tuples, tuples
-        of lists and ndarrays.
-    dtype : data-type, optional
-        By default, the data-type is inferred from the input data.
-    order : {'C', 'F'}, optional
-        Whether to use row-major ('C') or column-major ('F' for FORTRAN)
-        memory representation.  Defaults to 'C'.
+    a1, a2 : array_like
+        Input arrays.
 
     Returns
     -------
-    out : ndarray
-        Array interpretation of `a`.  No copy is performed if the input
-        is already an ndarray.  If `a` is a subclass of ndarray, a base
-        class ndarray is returned.
-
-    See Also
-    --------
-    asanyarray : Similar function which passes through subclasses.
-    ascontiguousarray : Convert input to a contiguous array.
-    asfarray : Convert input to a floating point ndarray.
-    asfortranarray : Convert input to an ndarray with column-major
-                     memory order.
-    asarray_chkfinite : Similar function which checks input for NaNs and Infs.
-    fromiter : Create an array from an iterator.
-    fromfunction : Construct an array by executing a function on grid
-                   positions.
+    out : bool
+        True if equivalent, False otherwise.
 
     Examples
     --------
-    Convert a list into an array:
-
-    >>> a = [1, 2]
-    >>> np.asarray(a)
-    array([1, 2])
-
-    Existing arrays are not copied:
-
-    >>> a = np.array([1, 2])
-    >>> np.asarray(a) is a
+    >>> np.array_equiv([1, 2], [1, 2])
     True
-
-    If `dtype` is set, array is copied only if dtype does not match:
-
-    >>> a = np.array([1, 2], dtype=np.float32)
-    >>> np.asarray(a, dtype=np.float32) is a
-    True
-    >>> np.asarray(a, dtype=np.float64) is a

From noreply at buildbot.pypy.org  Tue Oct 22 00:36:34 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue, 22 Oct 2013 00:36:34 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20131021223635.001B81C01F5@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67485:b396f95634ab
Date: 2013-10-17 11:59 -0700
http://bitbucket.org/pypy/pypy/changeset/b396f95634ab/

Log:	merge default

diff too long, truncating to 2000 out of 2633 lines

diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ b/lib_pypy/numpypy/core/arrayprint.py
@@ -247,10 +247,11 @@
     formatdict = {'bool' : _boolFormatter,
                   'int' : IntegerFormat(data),
                   'float' : FloatFormat(data, precision, suppress_small),
-                  'longfloat' : LongFloatFormat(precision),
+                  'longfloat' : FloatFormat(data, precision, suppress_small),
                   'complexfloat' : ComplexFormat(data, precision,
                                                  suppress_small),
-                  'longcomplexfloat' : LongComplexFormat(precision),
+                  'longcomplexfloat' : ComplexFormat(data, precision,
+                                                     suppress_small),
                   'datetime' : DatetimeFormat(data),
                   'timedelta' : TimedeltaFormat(data),
                   'numpystr' : repr_format,
diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -395,6 +395,9 @@
                   ('int_', 'long'),
                   ('uint', 'ulong'),
                   ('cfloat', 'cdouble'),
+                  ('longfloat', 'longdouble'),
+                  ('clongfloat', 'clongdouble'),
+                  ('longcomplex', 'clongdouble'),
                   ('bool_', 'bool'),
                   ('unicode_', 'unicode'),
                   ]
diff --git a/lib_pypy/numpypy/lib/__init__.py b/lib_pypy/numpypy/lib/__init__.py
--- a/lib_pypy/numpypy/lib/__init__.py
+++ b/lib_pypy/numpypy/lib/__init__.py
@@ -5,10 +5,12 @@
 from .function_base import *
 from .shape_base import *
 from .twodim_base import *
+from .ufunclike import *
 from .utils import *
 
 __all__ = ['math']
 __all__ += function_base.__all__
 __all__ += shape_base.__all__
 __all__ += twodim_base.__all__
+__all__ += ufunclike.__all__
 __all__ += utils.__all__
diff --git a/lib_pypy/numpypy/lib/ufunclike.py b/lib_pypy/numpypy/lib/ufunclike.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/lib/ufunclike.py
@@ -0,0 +1,177 @@
+"""
+Module of functions that are like ufuncs in acting on arrays and optionally
+storing results in an output array.
+
+"""
+from __future__ import division, absolute_import, print_function
+
+__all__ = ['fix', 'isneginf', 'isposinf']
+
+from ..core import numeric as nx
+
+def fix(x, y=None):
+    """
+    Round to nearest integer towards zero.
+
+    Round an array of floats element-wise to nearest integer towards zero.
+    The rounded values are returned as floats.
+
+    Parameters
+    ----------
+    x : array_like
+        An array of floats to be rounded
+    y : ndarray, optional
+        Output array
+
+    Returns
+    -------
+    out : ndarray of floats
+        The array of rounded numbers
+
+    See Also
+    --------
+    trunc, floor, ceil
+    around : Round to given number of decimals
+
+    Examples
+    --------
+    >>> np.fix(3.14)
+    3.0
+    >>> np.fix(3)
+    3.0
+    >>> np.fix([2.1, 2.9, -2.1, -2.9])
+    array([ 2.,  2., -2., -2.])
+
+    """
+    x = nx.asanyarray(x)
+    y1 = nx.floor(x)
+    y2 = nx.ceil(x)
+    if y is None:
+        y = nx.asanyarray(y1)
+    y[...] = nx.where(x >= 0, y1, y2)
+    return y
+
+def isposinf(x, y=None):
+    """
+    Test element-wise for positive infinity, return result as bool array.
+
+    Parameters
+    ----------
+    x : array_like
+        The input array.
+    y : array_like, optional
+        A boolean array with the same shape as `x` to store the result.
+
+    Returns
+    -------
+    y : ndarray
+        A boolean array with the same dimensions as the input.
+        If second argument is not supplied then a boolean array is returned
+        with values True where the corresponding element of the input is
+        positive infinity and values False where the element of the input is
+        not positive infinity.
+
+        If a second argument is supplied the result is stored there. If the
+        type of that array is a numeric type the result is represented as zeros
+        and ones, if the type is boolean then as False and True.
+        The return value `y` is then a reference to that array.
+
+    See Also
+    --------
+    isinf, isneginf, isfinite, isnan
+
+    Notes
+    -----
+    Numpy uses the IEEE Standard for Binary Floating-Point for Arithmetic
+    (IEEE 754).
+
+    Errors result if the second argument is also supplied when `x` is a
+    scalar input, or if first and second arguments have different shapes.
+
+    Examples
+    --------
+    >>> np.isposinf(np.PINF)
+    array(True, dtype=bool)
+    >>> np.isposinf(np.inf)
+    array(True, dtype=bool)
+    >>> np.isposinf(np.NINF)
+    array(False, dtype=bool)
+    >>> np.isposinf([-np.inf, 0., np.inf])
+    array([False, False,  True], dtype=bool)
+
+    >>> x = np.array([-np.inf, 0., np.inf])
+    >>> y = np.array([2, 2, 2])
+    >>> np.isposinf(x, y)
+    array([0, 0, 1])
+    >>> y
+    array([0, 0, 1])
+
+    """
+    if y is None:
+        x = nx.asarray(x)
+        y = nx.empty(x.shape, dtype=nx.bool_)
+    nx.logical_and(nx.isinf(x), ~nx.signbit(x), y)
+    return y
+
+def isneginf(x, y=None):
+    """
+    Test element-wise for negative infinity, return result as bool array.
+
+    Parameters
+    ----------
+    x : array_like
+        The input array.
+    y : array_like, optional
+        A boolean array with the same shape and type as `x` to store the
+        result.
+
+    Returns
+    -------
+    y : ndarray
+        A boolean array with the same dimensions as the input.
+        If second argument is not supplied then a numpy boolean array is
+        returned with values True where the corresponding element of the
+        input is negative infinity and values False where the element of
+        the input is not negative infinity.
+
+        If a second argument is supplied the result is stored there. If the
+        type of that array is a numeric type the result is represented as
+        zeros and ones, if the type is boolean then as False and True. The
+        return value `y` is then a reference to that array.
+
+    See Also
+    --------
+    isinf, isposinf, isnan, isfinite
+
+    Notes
+    -----
+    Numpy uses the IEEE Standard for Binary Floating-Point for Arithmetic
+    (IEEE 754).
+
+    Errors result if the second argument is also supplied when x is a scalar
+    input, or if first and second arguments have different shapes.
+
+    Examples
+    --------
+    >>> np.isneginf(np.NINF)
+    array(True, dtype=bool)
+    >>> np.isneginf(np.inf)
+    array(False, dtype=bool)
+    >>> np.isneginf(np.PINF)
+    array(False, dtype=bool)
+    >>> np.isneginf([-np.inf, 0., np.inf])
+    array([ True, False, False], dtype=bool)
+
+    >>> x = np.array([-np.inf, 0., np.inf])
+    >>> y = np.array([2, 2, 2])
+    >>> np.isneginf(x, y)
+    array([1, 0, 0])
+    >>> y
+    array([1, 0, 0])
+
+    """
+    if y is None:
+        x = nx.asarray(x)
+        y = nx.empty(x.shape, dtype=nx.bool_)
+    nx.logical_and(nx.isinf(x), nx.signbit(x), y)
+    return y
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -109,4 +109,5 @@
 .. branch: file-support-in-rpython
 make open() and friends rpython
 
-
+.. branch: incremental-gc
+Added the new incminimark GC which performs GC in incremental steps
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -1,5 +1,4 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module.micronumpy.interp_boxes import long_double_size, ENABLED_LONG_DOUBLE
 
 
 class MultiArrayModule(MixedModule):
@@ -64,6 +63,7 @@
         ("less_equal", "less_equal"),
         ("maximum", "maximum"),
         ("minimum", "minimum"),
+        ("mod", "mod"),
         ("multiply", "multiply"),
         ("negative", "negative"),
         ("not_equal", "not_equal"),
@@ -91,8 +91,6 @@
         ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
-        ('isneginf', 'isneginf'),
-        ('isposinf', 'isposinf'),
         ('isfinite', 'isfinite'),
         ('logical_and', 'logical_and'),
         ('logical_xor', 'logical_xor'),
diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -102,13 +102,10 @@
             dtype =  self.dtype.float_type
             return SliceArray(self.start + dtype.get_size(), strides,
                     backstrides, self.get_shape(), self, orig_array, dtype=dtype)
-        if self.dtype.is_flexible_type():
-            # numpy returns self for self.imag
-            return SliceArray(self.start, strides, backstrides,
-                    self.get_shape(), self, orig_array)
         impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides,
                              backstrides)
-        impl.fill(self.dtype.box(0))
+        if not self.dtype.is_flexible_type():
+            impl.fill(self.dtype.box(0))
         return impl
 
     def set_imag(self, space, orig_array, w_value):
@@ -129,7 +126,8 @@
                 idx = self.get_shape()[i] + idx
             if idx < 0 or idx >= self.get_shape()[i]:
                 raise operationerrfmt(space.w_IndexError,
-                      "index (%d) out of range (0<=index<%d", i, self.get_shape()[i],
+                      "index %d is out of bounds for axis %d with size %d",
+                      idx, i, self.get_shape()[i],
                 )
             item += idx * strides[i]
         return item
@@ -145,7 +143,8 @@
                 idx = shape[i] + idx
             if idx < 0 or idx >= shape[i]:
                 raise operationerrfmt(space.w_IndexError,
-                      "index (%d) out of range (0<=index<%d", i, shape[i],
+                      "index %d is out of bounds for axis %d with size %d",
+                      idx, i, self.get_shape()[i],
                 )
             item += idx * strides[i]
         return item
@@ -380,8 +379,8 @@
 
 class NonWritableArray(ConcreteArray):
     def descr_setitem(self, space, orig_array, w_index, w_value):
-        raise OperationError(space.w_RuntimeError, space.wrap(
-            "array is not writable"))
+        raise OperationError(space.w_ValueError, space.wrap(
+            "assignment destination is read-only"))
 
 
 class SliceArray(BaseConcreteArray):
diff --git a/pypy/module/micronumpy/constants.py b/pypy/module/micronumpy/constants.py
--- a/pypy/module/micronumpy/constants.py
+++ b/pypy/module/micronumpy/constants.py
@@ -1,4 +1,21 @@
+from pypy.interpreter.error import OperationError
 
-MODE_WRAP, MODE_RAISE, MODE_CLIP = range(3)
+MODE_CLIP, MODE_WRAP, MODE_RAISE = range(3)
 
-MODES = {'wrap': MODE_WRAP, 'raise': MODE_RAISE, 'clip': MODE_CLIP}
+def clipmode_converter(space, w_mode):
+    if space.is_none(w_mode):
+        return MODE_RAISE
+    if space.isinstance_w(w_mode, space.w_str):
+        mode = space.str_w(w_mode)
+        if mode.startswith('C') or mode.startswith('c'):
+            return MODE_CLIP
+        if mode.startswith('W') or mode.startswith('w'):
+            return MODE_WRAP
+        if mode.startswith('R') or mode.startswith('r'):
+            return MODE_RAISE
+    elif space.isinstance_w(w_mode, space.w_int):
+        mode = space.int_w(w_mode)
+        if MODE_CLIP <= mode <= MODE_RAISE:
+            return mode
+    raise OperationError(space.w_TypeError,
+                         space.wrap("clipmode not understood"))
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -1,10 +1,9 @@
-
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
-from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs
+from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs, constants
 from pypy.module.micronumpy.iter import Chunk, Chunks
 from pypy.module.micronumpy.strides import shape_agreement,\
      shape_agreement_multiple
-from pypy.module.micronumpy.constants import MODES
+from pypy.module.micronumpy.constants import clipmode_converter
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 
@@ -171,8 +170,7 @@
 def count_nonzero(space, w_obj):
     return space.wrap(loop.count_all_true(convert_to_array(space, w_obj)))
 
- at unwrap_spec(mode=str)
-def choose(space, w_arr, w_choices, w_out, mode):
+def choose(space, w_arr, w_choices, w_out, w_mode):
     arr = convert_to_array(space, w_arr)
     choices = [convert_to_array(space, w_item) for w_item
                in space.listview(w_choices)]
@@ -187,23 +185,16 @@
     shape = shape_agreement_multiple(space, choices + [w_out])
     out = interp_dtype.dtype_agreement(space, choices, shape, w_out)
     dtype = out.get_dtype()
-    if mode not in MODES:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("mode %s not known" % (mode,)))
-    loop.choose(space, arr, choices, shape, dtype, out, MODES[mode])
+    mode = clipmode_converter(space, w_mode)
+    loop.choose(space, arr, choices, shape, dtype, out, mode)
     return out
 
-
- at unwrap_spec(mode=str)
-def put(space, w_arr, w_indices, w_values, mode='raise'):
-    from pypy.module.micronumpy import constants
+def put(space, w_arr, w_indices, w_values, w_mode):
     from pypy.module.micronumpy.support import int_w
 
     arr = convert_to_array(space, w_arr)
+    mode = clipmode_converter(space, w_mode)
 
-    if mode not in constants.MODES:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("mode %s not known" % (mode,)))
     if not w_indices:
         raise OperationError(space.w_ValueError,
                              space.wrap("indice list cannot be empty"))
@@ -228,13 +219,13 @@
         index = int_w(space, idx)
 
         if index < 0 or index >= arr.get_size():
-            if constants.MODES[mode] == constants.MODE_RAISE:
-                raise OperationError(space.w_ValueError, space.wrap(
-                    "invalid entry in choice array"))
-            elif constants.MODES[mode] == constants.MODE_WRAP:
+            if mode == constants.MODE_RAISE:
+                raise OperationError(space.w_IndexError, space.wrap(
+                    "index %d is out of bounds for axis 0 with size %d" % (index, arr.get_size())))
+            elif mode == constants.MODE_WRAP:
                 index = index % arr.get_size()
             else:
-                assert constants.MODES[mode] == constants.MODE_CLIP
+                assert mode == constants.MODE_CLIP
                 if index < 0:
                     index = 0
                 else:
@@ -247,7 +238,6 @@
 
         arr.setitem(space, [index], dtype.coerce(space, value))
 
-
 def diagonal(space, arr, offset, axis1, axis2):
     shape = arr.get_shape()
     shapelen = len(shape)
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -20,14 +20,14 @@
 MIXIN_32 = (long_typedef,) if LONG_BIT == 32 else ()
 MIXIN_64 = (long_typedef,) if LONG_BIT == 64 else ()
 
-# Is this the proper place for this?
-ENABLED_LONG_DOUBLE = False
-long_double_size = rffi.sizeof_c_type('long double', ignore_errors=True)
+#long_double_size = rffi.sizeof_c_type('long double', ignore_errors=True)
+#import os
+#if long_double_size == 8 and os.name == 'nt':
+#    # this is a lie, or maybe a wish, MS fakes longdouble math with double
+#    long_double_size = 12
 
-import os
-if long_double_size == 8 and os.name == 'nt':
-    # this is a lie, or maybe a wish, MS fakes longdouble math with double
-    long_double_size = 12
+# hardcode to 8 for now (simulate using normal double) until long double works
+long_double_size = 8
 
 
 def new_dtype_getter(name):
@@ -230,11 +230,11 @@
 
     def descr_any(self, space):
         value = space.is_true(self)
-        return space.wrap(W_BoolBox(value))
+        return self.get_dtype(space).box(value)
 
     def descr_all(self, space):
         value = space.is_true(self)
-        return space.wrap(W_BoolBox(value))
+        return self.get_dtype(space).box(value)
 
     def descr_ravel(self, space):
         from pypy.module.micronumpy.base import convert_to_array
@@ -438,32 +438,18 @@
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex128")
     _COMPONENTS_BOX = W_Float64Box
 
-if ENABLED_LONG_DOUBLE and long_double_size == 12:
-    class W_Float96Box(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float96")
 
-    W_LongDoubleBox = W_Float96Box
+if long_double_size == 8:
+    W_FloatLongBox = W_Float64Box
+    W_ComplexLongBox = W_Complex128Box
 
-    class W_Complex192Box(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex192")
-        _COMPONENTS_BOX = W_Float96Box
+elif long_double_size in (12, 16):
+    class W_FloatLongBox(W_FloatingBox, PrimitiveBox):
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % (long_double_size * 8))
 
-    W_CLongDoubleBox = W_Complex192Box
-
-elif ENABLED_LONG_DOUBLE and long_double_size == 16:
-    class W_Float128Box(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float128")
-    W_LongDoubleBox = W_Float128Box
-
-    class W_Complex256Box(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex256")
-        _COMPONENTS_BOX = W_Float128Box
-
-    W_CLongDoubleBox = W_Complex256Box
-
-elif ENABLED_LONG_DOUBLE:
-    W_LongDoubleBox = W_Float64Box
-    W_CLongDoubleBox = W_Complex64Box
+    class W_ComplexLongBox(ComplexBox, W_ComplexFloatingBox):
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % (long_double_size * 16))
+        _COMPONENTS_BOX = W_FloatLongBox
 
 
 W_GenericBox.typedef = TypeDef("generic",
@@ -628,53 +614,53 @@
 
 W_Float16Box.typedef = TypeDef("float16", W_FloatingBox.typedef,
     __module__ = "numpypy",
-
     __new__ = interp2app(W_Float16Box.descr__new__.im_func),
     __reduce__ = interp2app(W_Float16Box.descr_reduce),
 )
 
 W_Float32Box.typedef = TypeDef("float32", W_FloatingBox.typedef,
     __module__ = "numpypy",
-
     __new__ = interp2app(W_Float32Box.descr__new__.im_func),
     __reduce__ = interp2app(W_Float32Box.descr_reduce),
 )
 
 W_Float64Box.typedef = TypeDef("float64", (W_FloatingBox.typedef, float_typedef),
     __module__ = "numpypy",
-
     __new__ = interp2app(W_Float64Box.descr__new__.im_func),
     __reduce__ = interp2app(W_Float64Box.descr_reduce),
 )
 
-if ENABLED_LONG_DOUBLE and long_double_size == 12:
-    W_Float96Box.typedef = TypeDef("float96", (W_FloatingBox.typedef),
+W_ComplexFloatingBox.typedef = TypeDef("complexfloating", W_InexactBox.typedef,
+    __module__ = "numpypy",
+)
+
+W_Complex64Box.typedef = TypeDef("complex64", (W_ComplexFloatingBox.typedef),
+    __module__ = "numpypy",
+    __new__ = interp2app(W_Complex64Box.descr__new__.im_func),
+    __reduce__ = interp2app(W_Complex64Box.descr_reduce),
+    real = GetSetProperty(W_ComplexFloatingBox .descr_get_real),
+    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
+)
+
+W_Complex128Box.typedef = TypeDef("complex128", (W_ComplexFloatingBox.typedef, complex_typedef),
+    __module__ = "numpypy",
+    __new__ = interp2app(W_Complex128Box.descr__new__.im_func),
+    __reduce__ = interp2app(W_Complex128Box.descr_reduce),
+    real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
+    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
+)
+
+if long_double_size in (12, 16):
+    W_FloatLongBox.typedef = TypeDef("float%d" % (long_double_size * 8), (W_FloatingBox.typedef),
         __module__ = "numpypy",
-        __reduce__ = interp2app(W_Float96Box.descr_reduce),
-
-        __new__ = interp2app(W_Float96Box.descr__new__.im_func),
+        __new__ = interp2app(W_FloatLongBox.descr__new__.im_func),
+        __reduce__ = interp2app(W_FloatLongBox.descr_reduce),
     )
 
-    W_Complex192Box.typedef = TypeDef("complex192", (W_ComplexFloatingBox.typedef, complex_typedef),
+    W_ComplexLongBox.typedef = TypeDef("complex%d" % (long_double_size * 16), (W_ComplexFloatingBox.typedef, complex_typedef),
         __module__ = "numpypy",
-        __new__ = interp2app(W_Complex192Box.descr__new__.im_func),
-        __reduce__ = interp2app(W_Complex192Box.descr_reduce),
-        real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
-        imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
-    )
-
-elif ENABLED_LONG_DOUBLE and long_double_size == 16:
-    W_Float128Box.typedef = TypeDef("float128", (W_FloatingBox.typedef),
-        __module__ = "numpypy",
-
-        __new__ = interp2app(W_Float128Box.descr__new__.im_func),
-        __reduce__ = interp2app(W_Float128Box.descr_reduce),
-    )
-
-    W_Complex256Box.typedef = TypeDef("complex256", (W_ComplexFloatingBox.typedef, complex_typedef),
-        __module__ = "numpypy",
-        __new__ = interp2app(W_Complex256Box.descr__new__.im_func),
-        __reduce__ = interp2app(W_Complex256Box.descr_reduce),
+        __new__ = interp2app(W_ComplexLongBox.descr__new__.im_func),
+        __reduce__ = interp2app(W_ComplexLongBox.descr_reduce),
         real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
         imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
     )
@@ -703,24 +689,3 @@
     __module__ = "numpypy",
     __new__ = interp2app(W_UnicodeBox.descr__new__unicode_box.im_func),
 )
-
-W_ComplexFloatingBox.typedef = TypeDef("complexfloating", W_InexactBox.typedef,
-    __module__ = "numpypy",
-)
-
-
-W_Complex128Box.typedef = TypeDef("complex128", (W_ComplexFloatingBox.typedef, complex_typedef),
-    __module__ = "numpypy",
-    __new__ = interp2app(W_Complex128Box.descr__new__.im_func),
-    __reduce__ = interp2app(W_Complex128Box.descr_reduce),
-    real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
-    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
-)
-
-W_Complex64Box.typedef = TypeDef("complex64", (W_ComplexFloatingBox.typedef),
-    __module__ = "numpypy",
-    __new__ = interp2app(W_Complex64Box.descr__new__.im_func),
-    __reduce__ = interp2app(W_Complex64Box.descr_reduce),
-    real = GetSetProperty(W_ComplexFloatingBox .descr_get_real),
-    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
-)
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -259,21 +259,22 @@
         builder_args = space.newtuple([space.wrap("%s%d" % (kind, elemsize)), space.wrap(0), space.wrap(1)])
 
         version = space.wrap(3)
-        order = space.wrap(byteorder_prefix if self.native else nonnative_byteorder_prefix)
         names = self.descr_get_names(space)
         values = self.descr_get_fields(space)
         if self.fields:
+            order = space.wrap('|')
             #TODO: Implement this when subarrays are implemented
             subdescr = space.w_None
-            #TODO: Change this when alignment is implemented :
             size = 0
             for key in self.fields:
                 dtype = self.fields[key][1]
                 assert isinstance(dtype, W_Dtype)
                 size += dtype.get_size()
             w_size = space.wrap(size)
+            #TODO: Change this when alignment is implemented
             alignment = space.wrap(1)
         else:
+            order = space.wrap(byteorder_prefix if self.native else nonnative_byteorder_prefix)
             subdescr = space.w_None
             w_size = space.wrap(-1)
             alignment = space.wrap(-1)
@@ -542,15 +543,11 @@
             char="I",
             w_box_type=space.gettypefor(interp_boxes.W_UInt32Box),
         )
-        if LONG_BIT == 32:
-            name = "int32"
-        elif LONG_BIT == 64:
-            name = "int64"
         self.w_longdtype = W_Dtype(
             types.Long(),
             num=7,
             kind=SIGNEDLTR,
-            name=name,
+            name="int%d" % LONG_BIT,
             char="l",
             w_box_type=space.gettypefor(interp_boxes.W_LongBox),
             alternate_constructors=[space.w_int,
@@ -563,7 +560,7 @@
             types.ULong(),
             num=8,
             kind=UNSIGNEDLTR,
-            name="u" + name,
+            name="uint%d" % LONG_BIT,
             char="L",
             w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
             alternate_constructors=[ space.gettypefor(interp_boxes.W_UnsignedIntegerBox),
@@ -607,6 +604,15 @@
                                    ],
             aliases=["float", "double"],
         )
+        self.w_floatlongdtype = W_Dtype(
+            types.FloatLong(),
+            num=13,
+            kind=FLOATINGLTR,
+            name="float%d" % (interp_boxes.long_double_size * 8),
+            char="g",
+            w_box_type=space.gettypefor(interp_boxes.W_FloatLongBox),
+            aliases=["longdouble", "longfloat"],
+        )
         self.w_complex64dtype = W_ComplexDtype(
             types.Complex64(),
             num=14,
@@ -627,57 +633,16 @@
             aliases=["complex"],
             float_type = self.w_float64dtype,
         )
-        if interp_boxes.ENABLED_LONG_DOUBLE and interp_boxes.long_double_size == 12:
-            self.w_float96dtype = W_Dtype(
-                types.Float96(),
-                num=13,
-                kind=FLOATINGLTR,
-                name="float96",
-                char="g",
-                w_box_type=space.gettypefor(interp_boxes.W_Float96Box),
-                aliases=["longdouble", "longfloat"],
-            )
-            self.w_complex192dtype = W_ComplexDtype(
-                types.Complex192(),
-                num=16,
-                kind=COMPLEXLTR,
-                name="complex192",
-                char="G",
-                w_box_type = space.gettypefor(interp_boxes.W_Complex192Box),
-                alternate_constructors=[space.w_complex],
-                aliases=["clongdouble", "clongfloat"],
-                float_type = self.w_float96dtype,
-            )
-            self.w_longdouble = self.w_float96dtype
-            self.w_clongdouble = self.w_complex192dtype
-        elif interp_boxes.ENABLED_LONG_DOUBLE and interp_boxes.long_double_size == 16:
-            self.w_float128dtype = W_Dtype(
-                types.Float128(),
-                num=13,
-                kind=FLOATINGLTR,
-                name="float128",
-                char="g",
-                w_box_type=space.gettypefor(interp_boxes.W_Float128Box),
-                aliases=["longdouble", "longfloat"],
-            )
-            self.w_complex256dtype = W_ComplexDtype(
-                types.Complex256(),
-                num=16,
-                kind=COMPLEXLTR,
-                name="complex256",
-                char="G",
-                w_box_type = space.gettypefor(interp_boxes.W_Complex256Box),
-                alternate_constructors=[space.w_complex],
-                aliases=["clongdouble", "clongfloat"],
-                float_type = self.w_float128dtype,
-            )
-            self.w_longdouble = self.w_float128dtype
-            self.w_clongdouble = self.w_complex256dtype
-        elif interp_boxes.ENABLED_LONG_DOUBLE:
-            self.w_float64dtype.aliases += ["longdouble", "longfloat"]
-            self.w_complex128dtype.aliases += ["clongdouble", "clongfloat"]
-            self.w_longdouble = self.w_float64dtype
-            self.w_clongdouble = self.w_complex128dtype
+        self.w_complexlongdtype = W_ComplexDtype(
+            types.ComplexLong(),
+            num=16,
+            kind=COMPLEXLTR,
+            name="complex%d" % (interp_boxes.long_double_size * 16),
+            char="G",
+            w_box_type = space.gettypefor(interp_boxes.W_ComplexLongBox),
+            aliases=["clongdouble", "clongfloat"],
+            float_type = self.w_floatlongdtype,
+        )
         self.w_stringdtype = W_Dtype(
             types.StringType(0),
             num=18,
@@ -750,21 +715,18 @@
             char=UINTPLTR,
             w_box_type = space.gettypefor(uintp_box),
         )
-        float_dtypes = [self.w_float16dtype,
-                self.w_float32dtype, self.w_float64dtype,
-                ]
-        complex_dtypes =  [self.w_complex64dtype, self.w_complex128dtype]
-        if interp_boxes.ENABLED_LONG_DOUBLE:
-            float_dtypes.append(self.w_longdouble)
-            complex_dtypes.append(self.w_clongdouble)
+        float_dtypes = [self.w_float16dtype, self.w_float32dtype,
+                        self.w_float64dtype, self.w_floatlongdtype]
+        complex_dtypes = [self.w_complex64dtype, self.w_complex128dtype,
+                          self.w_complexlongdtype]
         self.builtin_dtypes = [
             self.w_booldtype,
             self.w_int8dtype, self.w_uint8dtype,
             self.w_int16dtype, self.w_uint16dtype,
             self.w_longdtype, self.w_ulongdtype,
             self.w_int32dtype, self.w_uint32dtype,
-            self.w_int64dtype, self.w_uint64dtype] + \
-            float_dtypes + complex_dtypes + [
+            self.w_int64dtype, self.w_uint64dtype,
+            ] + float_dtypes + complex_dtypes + [
             self.w_stringdtype, self.w_unicodedtype, self.w_voiddtype,
             self.w_intpdtype, self.w_uintpdtype,
         ]
@@ -818,6 +780,7 @@
             'STRING': self.w_stringdtype,
             'CFLOAT': self.w_complex64dtype,
             'CDOUBLE': self.w_complex128dtype,
+            'CLONGDOUBLE': self.w_complexlongdtype,
             #'DATETIME',
             'UINT': self.w_uint32dtype,
             'INTP': self.w_intpdtype,
@@ -827,13 +790,11 @@
             #'TIMEDELTA',
             'INT': self.w_int32dtype,
             'DOUBLE': self.w_float64dtype,
+            'LONGDOUBLE': self.w_floatlongdtype,
             'USHORT': self.w_uint16dtype,
             'FLOAT': self.w_float32dtype,
             'BOOL': self.w_booldtype,
         }
-        if interp_boxes.ENABLED_LONG_DOUBLE:
-            typeinfo_full['LONGDOUBLE'] = self.w_longdouble
-            typeinfo_full['CLONGDOUBLE'] = self.w_clongdouble
 
         typeinfo_partial = {
             'Generic': interp_boxes.W_GenericBox,
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,4 +1,3 @@
-
 from pypy.interpreter.error import operationerrfmt, OperationError
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -14,7 +13,7 @@
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy import loop
 from pypy.module.micronumpy.dot import match_dot_shapes
-from pypy.module.micronumpy.interp_arrayops import repeat, choose
+from pypy.module.micronumpy.interp_arrayops import repeat, choose, put
 from pypy.module.micronumpy.arrayimpl import scalar
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rlib import jit
@@ -421,8 +420,8 @@
                                      [0] * len(self.get_shape()))
                 assert isinstance(w_obj, interp_boxes.W_GenericBox)
                 return w_obj.item(space)
-            raise OperationError(space.w_IndexError,
-                                 space.wrap("index out of bounds"))
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("can only convert an array of size 1 to a Python scalar"))
         if space.isinstance_w(w_arg, space.w_int):
             if self.is_scalar():
                 raise OperationError(space.w_IndexError,
@@ -509,9 +508,8 @@
             loop.byteswap(self.implementation, w_res.implementation)
             return w_res
 
-    @unwrap_spec(mode=str)
-    def descr_choose(self, space, w_choices, w_out=None, mode='raise'):
-        return choose(space, self, w_choices, w_out, mode)
+    def descr_choose(self, space, w_choices, w_out=None, w_mode=None):
+        return choose(space, self, w_choices, w_out, w_mode)
 
     def descr_clip(self, space, w_min, w_max, w_out=None):
         if space.is_none(w_out):
@@ -550,6 +548,12 @@
         return interp_arrayops.diagonal(space, self.implementation, offset,
                                         axis1, axis2)
 
+    @unwrap_spec(offset=int, axis1=int, axis2=int)
+    def descr_trace(self, space, offset=0, axis1=0, axis2=1,
+                    w_dtype=None, w_out=None):
+        diag = self.descr_diagonal(space, offset, axis1, axis2)
+        return diag.descr_sum(space, w_axis=space.wrap(-1), w_dtype=w_dtype, w_out=w_out)
+
     def descr_dump(self, space, w_file):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "dump not implemented yet"))
@@ -584,10 +588,8 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "ptp (peak to peak) not implemented yet"))
 
-    @unwrap_spec(mode=str)
-    def descr_put(self, space, w_indices, w_values, mode='raise'):
-        from pypy.module.micronumpy.interp_arrayops import put
-        put(space, self, w_indices, w_values, mode)
+    def descr_put(self, space, w_indices, w_values, w_mode=None):
+        put(space, self, w_indices, w_values, w_mode)
 
     def descr_resize(self, space, w_new_shape, w_refcheck=True):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -653,11 +655,6 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "tofile not implemented yet"))
 
-    def descr_trace(self, space, w_offset=0, w_axis1=0, w_axis2=1,
-                    w_dtype=None, w_out=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "trace not implemented yet"))
-
     def descr_view(self, space, w_dtype=None, w_type=None) :
         if not w_type and w_dtype:
             try:
@@ -845,7 +842,7 @@
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False,
                            cumultative=False):
-        def impl(self, space, w_axis=None, w_out=None, w_dtype=None):
+        def impl(self, space, w_axis=None, w_dtype=None, w_out=None):
             if space.is_none(w_out):
                 out = None
             elif not isinstance(w_out, W_NDimArray):
@@ -1153,6 +1150,7 @@
     round    = interp2app(W_NDimArray.descr_round),
     data     = GetSetProperty(W_NDimArray.descr_get_data),
     diagonal = interp2app(W_NDimArray.descr_diagonal),
+    trace = interp2app(W_NDimArray.descr_trace),
     view = interp2app(W_NDimArray.descr_view),
 
     ctypes = GetSetProperty(W_NDimArray.descr_get_ctypes), # XXX unimplemented
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -16,23 +16,24 @@
 def done_if_false(dtype, val):
     return not dtype.itemtype.bool(val)
 
+
 class W_Ufunc(W_Root):
     _attrs_ = ["name", "promote_to_float", "promote_bools", "identity",
-               "allow_complex", "complex_to_float"]
+               "allow_bool", "allow_complex", "complex_to_float"]
     _immutable_fields_ = ["promote_to_float", "promote_bools", "name",
-            "allow_complex", "complex_to_float"]
+            "allow_bool", "allow_complex", "complex_to_float"]
 
     def __init__(self, name, promote_to_float, promote_bools, identity,
-                 int_only, allow_complex, complex_to_float):
+                 int_only, allow_bool, allow_complex, complex_to_float):
         self.name = name
         self.promote_to_float = promote_to_float
         self.promote_bools = promote_bools
+        self.identity = identity
+        self.int_only = int_only
+        self.allow_bool = allow_bool
         self.allow_complex = allow_complex
         self.complex_to_float = complex_to_float
 
-        self.identity = identity
-        self.int_only = int_only
-
     def descr_repr(self, space):
         return space.wrap("<ufunc '%s'>" % self.name)
 
@@ -259,10 +260,10 @@
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
         identity=None, bool_result=False, int_only=False,
-        allow_complex=True, complex_to_float=False):
+        allow_bool=True, allow_complex=True, complex_to_float=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity,
-                         int_only, allow_complex, complex_to_float)
+                         int_only, allow_bool, allow_complex, complex_to_float)
         self.func = func
         self.bool_result = bool_result
 
@@ -274,17 +275,19 @@
             if space.is_w(out, space.w_None):
                 out = None
         w_obj = convert_to_array(space, w_obj)
-        if w_obj.get_dtype().is_flexible_type():
+        dtype = w_obj.get_dtype()
+        if dtype.is_flexible_type():
             raise OperationError(space.w_TypeError,
                       space.wrap('Not implemented for this type'))
-        if self.int_only and not w_obj.get_dtype().is_int_type():
+        if (self.int_only and not dtype.is_int_type() or
+                not self.allow_bool and dtype.is_bool_type() or
+                not self.allow_complex and dtype.is_complex_type()):
             raise OperationError(space.w_TypeError, space.wrap(
                 "ufunc %s not supported for the input type" % self.name))
         calc_dtype = find_unaryop_result_dtype(space,
                                   w_obj.get_dtype(),
                                   promote_to_float=self.promote_to_float,
-                                  promote_bools=self.promote_bools,
-                                  allow_complex=self.allow_complex)
+                                  promote_bools=self.promote_bools)
         if out is not None:
             if not isinstance(out, W_NDimArray):
                 raise OperationError(space.w_TypeError, space.wrap(
@@ -324,10 +327,10 @@
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
         identity=None, comparison_func=False, int_only=False,
-        allow_complex=True, complex_to_float=False):
+        allow_bool=True, allow_complex=True, complex_to_float=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity,
-                         int_only, allow_complex, complex_to_float)
+                         int_only, allow_bool, allow_complex, complex_to_float)
         self.func = func
         self.comparison_func = comparison_func
         if name == 'logical_and':
@@ -375,16 +378,14 @@
                 w_rdtype = w_ldtype
             elif w_lhs.is_scalar() and not w_rhs.is_scalar():
                 w_ldtype = w_rdtype
+        if (self.int_only and (not w_ldtype.is_int_type() or not w_rdtype.is_int_type()) or
+                not self.allow_bool and (w_ldtype.is_bool_type() or w_rdtype.is_bool_type()) or
+                not self.allow_complex and (w_ldtype.is_complex_type() or w_rdtype.is_complex_type())):
+            raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
         calc_dtype = find_binop_result_dtype(space,
             w_ldtype, w_rdtype,
-            int_only=self.int_only,
             promote_to_float=self.promote_to_float,
-            promote_bools=self.promote_bools,
-            allow_complex=self.allow_complex,
-            )
-        if self.int_only and not calc_dtype.is_int_type():
-            raise OperationError(space.w_TypeError, space.wrap(
-                "ufunc '%s' not supported for the input types" % self.name))
+            promote_bools=self.promote_bools)
         if space.is_none(w_out):
             out = None
         elif not isinstance(w_out, W_NDimArray):
@@ -431,14 +432,10 @@
 
 
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
-    promote_bools=False, int_only=False, allow_complex=True):
+        promote_bools=False):
     # dt1.num should be <= dt2.num
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
-    if int_only and (not dt1.is_int_type() or not dt2.is_int_type()):
-        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
-    if not allow_complex and (dt1.is_complex_type() or dt2.is_complex_type()):
-        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
     # Some operations promote op(bool, bool) to return int8, rather than bool
     if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
@@ -449,8 +446,8 @@
             return interp_dtype.get_dtype_cache(space).w_complex64dtype
         elif dt2.num == 15:
             return interp_dtype.get_dtype_cache(space).w_complex128dtype
-        elif interp_boxes.ENABLED_LONG_DOUBLE and dt2.num == 16:
-            return interp_dtype.get_dtype_cache(space).w_clongdouble
+        elif dt2.num == 16:
+            return interp_dtype.get_dtype_cache(space).w_complexlongdtype
         else:
             raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
 
@@ -507,14 +504,11 @@
         dtypenum += 2
         return interp_dtype.get_dtype_cache(space).dtypes_by_num[dtypenum]
 
-
 @jit.unroll_safe
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
-    promote_bools=False, promote_to_largest=False, allow_complex=True):
+        promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
-    if not allow_complex and (dt.is_complex_type()):
-        raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
     if promote_to_float:
         if dt.kind == interp_dtype.FLOATINGLTR or dt.kind==interp_dtype.COMPLEXLTR:
             return dt
@@ -535,7 +529,6 @@
             assert False
     return dt
 
-
 def find_dtype_for_scalar(space, w_obj, current_guess=None):
     bool_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
     long_dtype = interp_dtype.get_dtype_cache(space).w_longdtype
@@ -583,7 +576,6 @@
         'unable to create dtype from objects, ' '"%T" instance not supported',
         w_obj)
 
-
 def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func,
                        bool_result):
     dtype_cache = interp_dtype.get_dtype_cache(space)
@@ -601,6 +593,7 @@
             return res
     return func_with_new_name(impl, ufunc_name)
 
+
 class UfuncState(object):
     def __init__(self, space):
         "NOT_RPYTHON"
@@ -630,10 +623,6 @@
             ("greater_equal", "ge", 2, {"comparison_func": True}),
             ("isnan", "isnan", 1, {"bool_result": True}),
             ("isinf", "isinf", 1, {"bool_result": True}),
-            ("isneginf", "isneginf", 1, {"bool_result": True,
-                                         "allow_complex": False}),
-            ("isposinf", "isposinf", 1, {"bool_result": True,
-                                         "allow_complex": False}),
             ("isfinite", "isfinite", 1, {"bool_result": True}),
 
             ('logical_and', 'logical_and', 2, {'comparison_func': True,
@@ -653,7 +642,7 @@
             ("negative", "neg", 1),
             ("absolute", "abs", 1, {"complex_to_float": True}),
             ("rint", "rint", 1),
-            ("sign", "sign", 1, {"promote_bools": True}),
+            ("sign", "sign", 1, {"allow_bool": False}),
             ("signbit", "signbit", 1, {"bool_result": True,
                                        "allow_complex": False}),
             ("reciprocal", "reciprocal", 1),
@@ -708,6 +697,7 @@
                                        "allow_complex": False}),
             ("logaddexp2", "logaddexp2", 2, {"promote_to_float": True,
                                        "allow_complex": False}),
+
             ("ones_like", "ones_like", 1),
             ("zeros_like", "zeros_like", 1),
         ]:
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
--- a/pypy/module/micronumpy/test/test_arrayops.py
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -153,5 +153,12 @@
         a = arange(5)
         a.put(22, -5, mode='wrap')
         assert (a == array([0, 1, -5, 3, 4])).all()
-        raises(ValueError, "arange(5).put(22, -5, mode='raise')")
-        raises(ValueError, "arange(5).put(22, -5, mode='wrongmode')")
+        raises(IndexError, "arange(5).put(22, -5, mode='raise')")
+        raises(IndexError, "arange(5).put(22, -5, mode=2)")  # raise
+        a.put(22, -10, mode='wrongmode_starts_with_w_so_wrap')
+        assert (a == array([0, 1, -10, 3, 4])).all()
+        a.put(22, -15, mode='cccccccc')
+        assert (a == array([0, 1, -10, 3, -15])).all()
+        a.put(23, -1, mode=1)  # wrap
+        assert (a == array([0, 1, -10, -1, -15])).all()
+        raises(TypeError, "arange(5).put(22, -5, mode='zzzz')")  # unrecognized mode
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -196,13 +196,7 @@
         raises(TypeError, signbit, complex(1,1))
 
     def test_reciprocal(self):
-        from numpypy import array, reciprocal, complex64, complex128
-        c_and_relerr = [(complex64, 2e-7), (complex128, 2e-15)]
-        try:
-            from numpypy import clongdouble
-            c_and_relerr.append((clongdouble, 2e-30))
-        except:
-            pass # no longdouble yet
+        from numpypy import array, reciprocal, complex64, complex128, clongdouble
         inf = float('inf')
         nan = float('nan')
         #complex
@@ -217,7 +211,7 @@
                     complex(-r, i),
                     -0j, 0j, cnan,
                     cnan, cnan, cnan]
-        for c, rel_err in c_and_relerr:
+        for c, rel_err in ((complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)):
             actual = reciprocal(array([orig], dtype=c))
             for b, a, e in zip(orig, actual, expected):
                 assert (a[0].real - e.real) < rel_err
@@ -237,18 +231,12 @@
         raises(TypeError, copysign, a, b)
 
     def test_exp2(self):
-        from numpypy import array, exp2, complex128, complex64
-        c_and_relerr = [(complex64, 2e-7), (complex128, 2e-15)]
-        try:
-            from numpypy import clongdouble
-            c_and_relerr.append((clongdouble, 2e-30))
-        except:
-            pass # no longdouble yet
+        from numpypy import array, exp2, complex128, complex64, clongdouble
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
         cmpl = complex
-        for c,rel_err in c_and_relerr:
+        for c, rel_err in ((complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)):
             a = [cmpl(-5., 0), cmpl(-5., -5.), cmpl(-5., 5.),
                        cmpl(0., -5.), cmpl(0., 0.), cmpl(0., 5.),
                        cmpl(-0., -5.), cmpl(-0., 0.), cmpl(-0., 5.),
@@ -279,12 +267,12 @@
 
     def test_expm1(self):
         import math, cmath
-        from numpypy import array, expm1, complex128, complex64
+        from numpypy import array, expm1, complex128, complex64, clongdouble
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
         cmpl = complex
-        for c,rel_err in ((complex128, 2e-15), (complex64, 1e-7)):
+        for c, rel_err in ((complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)):
             a = [cmpl(-5., 0), cmpl(-5., -5.), cmpl(-5., 5.),
                        cmpl(0., -5.), cmpl(0., 0.), cmpl(0., 5.),
                        cmpl(-0., -5.), cmpl(-0., 0.), cmpl(-0., 5.),
@@ -508,15 +496,9 @@
     def test_basic(self):
         from numpypy import (complex128, complex64, add, array, dtype,
             subtract as sub, multiply, divide, negative, absolute as abs,
-            floor_divide, real, imag, sign)
+            floor_divide, real, imag, sign, clongdouble)
         from numpypy import (equal, not_equal, greater, greater_equal, less,
                 less_equal, isnan)
-        complex_dtypes = [complex64, complex128]
-        try:
-            from numpypy import clongfloat
-            complex_dtypes.append(clongfloat)
-        except:
-            pass
         assert real(4.0) == 4.0
         assert imag(0.0) == 0.0
         a = array([complex(3.0, 4.0)])
@@ -545,8 +527,7 @@
         assert str(a.real) == 'abc'
         # numpy imag for flexible types returns self
         assert str(a.imag) == 'abc'
-        for complex_ in complex_dtypes:
-
+        for complex_ in complex64, complex128, clongdouble:
             O = complex(0, 0)
             c0 = complex_(complex(2.5, 0))
             c1 = complex_(complex(1, 2))
@@ -577,7 +558,6 @@
             assert negative(complex(1,1)) == complex(-1, -1)
             assert negative(complex(0, 0)) == 0
 
-
             assert multiply(1, c1) == c1
             assert multiply(2, c2) == complex(6, 8)
             assert multiply(c1, c2) == complex(-5, 10)
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -45,6 +45,13 @@
         raises(TypeError, lambda: dtype("int8") == 3)
         assert dtype(bool) == bool
 
+    def test_dtype_aliases(self):
+        from numpypy import dtype
+        assert dtype('longfloat').num in (12, 13)
+        assert dtype('longdouble').num in (12, 13)
+        assert dtype('clongfloat').num in (15, 16)
+        assert dtype('clongdouble').num in (15, 16)
+
     def test_dtype_with_types(self):
         from numpypy import dtype
 
@@ -152,6 +159,8 @@
             '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd',
             'e'
         ]
+        if array([0], dtype='longdouble').itemsize > 8:
+            types += ['g', 'G']
         a = array([True], '?')
         for t in types:
             assert (a + array([0], t)).dtype is dtype(t)
@@ -267,6 +276,7 @@
             (numpy.float16, 10.),
             (numpy.float32, 2.0),
             (numpy.float64, 4.32),
+            (numpy.longdouble, 4.32),
         ]:
             assert hash(tp(value)) == hash(value)
 
@@ -532,6 +542,20 @@
         from math import isnan
         assert isnan(numpy.float32(None))
         assert isnan(numpy.float64(None))
+        assert isnan(numpy.longdouble(None))
+
+    def test_longfloat(self):
+        import numpypy as numpy
+        # it can be float96 or float128
+        if numpy.longfloat != numpy.float64:
+            assert numpy.longfloat.mro()[1:] == [numpy.floating,
+                                       numpy.inexact, numpy.number,
+                                       numpy.generic, object]
+        a = numpy.array([1, 2, 3], numpy.longdouble)
+        assert type(a[1]) is numpy.longdouble
+        assert numpy.float64(12) == numpy.longdouble(12)
+        assert numpy.float64(12) == numpy.longfloat(12)
+        raises(ValueError, numpy.longfloat, '23.2df')
 
     def test_complex_floating(self):
         import numpypy as numpy
@@ -712,19 +736,38 @@
         assert dtype('i4').isnative == True
         assert dtype('>i8').isnative == False
 
-    def test_any_all(self):
+    def test_any_all_nonzero(self):
         import numpypy as numpy
         x = numpy.bool_(True)
         assert x.any()
         assert x.all()
+        assert x.__nonzero__()
+        assert isinstance(x.any(), numpy.bool_)
+        assert isinstance(x.__nonzero__(), bool)
         x = numpy.bool_(False)
         assert not x.any()
         assert not x.all()
-        #
+        assert not x.__nonzero__()
+        assert isinstance(x.any(), numpy.bool_)
+        assert isinstance(x.__nonzero__(), bool)
         x = numpy.float64(0)
         assert not x.any()
         assert not x.all()
-        assert isinstance(x.any(), numpy.bool_)
+        assert not x.__nonzero__()
+        assert isinstance(x.any(), numpy.float64)
+        assert isinstance(x.__nonzero__(), bool)
+        x = numpy.complex128(0)
+        assert not x.any()
+        assert not x.all()
+        assert not x.__nonzero__()
+        assert isinstance(x.any(), numpy.complex128)
+        assert isinstance(x.__nonzero__(), bool)
+        x = numpy.complex128(0+1j)
+        assert x.any()
+        assert x.all()
+        assert x.__nonzero__()
+        assert isinstance(x.any(), numpy.complex128)
+        assert isinstance(x.__nonzero__(), bool)
 
     def test_ravel(self):
         from numpypy import float64, int8, array
@@ -737,7 +780,6 @@
         assert (x == array(42)).all()
 
 
-
 class AppTestStrUnicodeDtypes(BaseNumpyAppTest):
     def test_str_unicode(self):
         skip('numpypy differs from numpy')
@@ -842,7 +884,7 @@
         from cPickle import loads, dumps
 
         d = dtype([("x", "int32"), ("y", "int32"), ("z", "int32"), ("value", float)])
-        assert d.__reduce__() == (dtype, ('V20', 0, 1), (3, '<', None, ('x', 'y', 'z', 'value'), {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0), 'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12)}, 20, 1, 0))
+        assert d.__reduce__() == (dtype, ('V20', 0, 1), (3, '|', None, ('x', 'y', 'z', 'value'), {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0), 'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12)}, 20, 1, 0))
 
         new_d = loads(dumps(d))
 
@@ -895,6 +937,12 @@
         a = array([1, 2, 3], dtype=self.non_native_prefix + 'f2')
         assert a[0] == 1
         assert (a + a)[1] == 4
+        a = array([1, 2, 3], dtype=self.non_native_prefix + 'g') # longdouble
+        assert a[0] == 1
+        assert (a + a)[1] == 4
+        a = array([1, 2, 3], dtype=self.non_native_prefix + 'G') # clongdouble
+        assert a[0] == 1
+        assert (a + a)[1] == 4
 
 class AppTestPyPyOnly(BaseNumpyAppTest):
     def setup_class(cls):
@@ -913,84 +961,6 @@
         assert typeinfo['CDOUBLE'] == ('D', 15, 128, 8, complex128)
         assert typeinfo['HALF'] == ('e', 23, 16, 2, float16)
 
-class AppTestNoLongDoubleDtypes(BaseNumpyAppTest):
-    def setup_class(cls):
-        from pypy.module.micronumpy import Module
-        if Module.interpleveldefs.get('longfloat', None):
-            py.test.skip('longdouble exists, skip these tests')
-        if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("pypy only test for no longdouble support")
-        BaseNumpyAppTest.setup_class.im_func(cls)
-
-    def test_nolongfloat(self):
-        import numpypy
-        from numpypy import dtype
-        assert not getattr(numpypy, 'longdouble', False)
-        assert not getattr(numpypy, 'float128', False)
-        assert not getattr(numpypy, 'float96', False)
-        raises(TypeError, dtype, 'longdouble')
-        raises(TypeError, dtype, 'clongdouble')
-        raises(TypeError, dtype, 'longfloat')
-        raises(TypeError, dtype, 'clongfloat')
-        raises(TypeError, dtype, 'float128')
-        raises(TypeError, dtype, 'float96')
-
-class AppTestLongDoubleDtypes(BaseNumpyAppTest):
-    def setup_class(cls):
-        from pypy.module.micronumpy import Module
-        print dir(Module.interpleveldefs)
-        if not Module.interpleveldefs.get('longfloat', None):
-            py.test.skip('no longdouble types yet')
-        BaseNumpyAppTest.setup_class.im_func(cls)
-
-    def test_longfloat(self):
-        import numpypy as numpy
-        # it can be float96 or float128
-        if numpy.longfloat != numpy.float64:
-            assert numpy.longfloat.mro()[1:] == [numpy.floating,
-                                       numpy.inexact, numpy.number,
-                                       numpy.generic, object]
-        a = numpy.array([1, 2, 3], numpy.longdouble)
-        assert type(a[1]) is numpy.longdouble
-        assert numpy.float64(12) == numpy.longdouble(12)
-        assert numpy.float64(12) == numpy.longfloat(12)
-        raises(ValueError, numpy.longfloat, '23.2df')
-
-    def test_dtype_aliases(self):
-        from numpypy import dtype
-        assert dtype('longfloat').num in (12, 13)
-        assert dtype('longdouble').num in (12, 13)
-        assert dtype('clongfloat').num in (15, 16)
-        assert dtype('clongdouble').num in (15, 16)
-
-    def test_bool_binop_types(self):
-        from numpypy import array, dtype
-        types = ['g', 'G']
-        a = array([True], '?')
-        for t in types:
-            assert (a + array([0], t)).dtype is dtype(t)
-
-    def test_hash(self):
-        import numpypy as numpy
-        for tp, value in [
-            (numpy.longdouble, 4.32),
-        ]:
-            assert hash(tp(value)) == hash(value)
-
-    def test_float_None(self):
-        import numpypy as numpy
-        from math import isnan
-        assert isnan(numpy.longdouble(None))
-
-    def test_non_native(self):
-        from numpypy import array
-        a = array([1, 2, 3], dtype=self.non_native_prefix + 'g') # longdouble
-        assert a[0] == 1
-        assert (a + a)[1] == 4
-        a = array([1, 2, 3], dtype=self.non_native_prefix + 'G') # clongdouble
-        assert a[0] == 1
-        assert (a + a)[1] == 4
-
 class AppTestObjectDtypes(BaseNumpyAppTest):
     def test_scalar_from_object(self):
         from numpypy import array
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -365,6 +365,26 @@
         assert b[0] == 1+0j
         assert b.dtype is dtype(complex)
 
+    def test_arange(self):
+        from numpypy import arange, dtype
+        a = arange(3)
+        assert (a == [0, 1, 2]).all()
+        assert a.dtype is dtype(int)
+        a = arange(3.0)
+        assert (a == [0., 1., 2.]).all()
+        assert a.dtype is dtype(float)
+        a = arange(3, 7)
+        assert (a == [3, 4, 5, 6]).all()
+        assert a.dtype is dtype(int)
+        a = arange(3, 7, 2)
+        assert (a == [3, 5]).all()
+        a = arange(3, dtype=float)
+        assert (a == [0., 1., 2.]).all()
+        assert a.dtype is dtype(float)
+        a = arange(0, 0.8, 0.1)
+        assert len(a) == 8
+        assert arange(False, True, True).dtype is dtype(int)
+
     def test_copy(self):
         from numpypy import arange, array
         a = arange(5)
@@ -430,24 +450,17 @@
 
     def test_getitem_obj_index(self):
         from numpypy import arange
-
         a = arange(10)
-
         assert a[self.CustomIndexObject(1)] == 1
 
     def test_getitem_obj_prefer_index_to_int(self):
         from numpypy import arange
-
         a = arange(10)
-
-
         assert a[self.CustomIndexIntObject(0, 1)] == 0
 
     def test_getitem_obj_int(self):
         from numpypy import arange
-
         a = arange(10)
-
         assert a[self.CustomIntObject(1)] == 1
 
     def test_setitem(self):
@@ -469,7 +482,6 @@
         assert a[1] == -0.005
         assert a[2] == -0.005
 
-
     def test_setitem_tuple(self):
         from numpypy import array
         a = array(range(5))
@@ -483,27 +495,20 @@
 
     def test_setitem_obj_index(self):
         from numpypy import arange
-
         a = arange(10)
-
         a[self.CustomIndexObject(1)] = 100
         assert a[1] == 100
 
     def test_setitem_obj_prefer_index_to_int(self):
         from numpypy import arange
-
         a = arange(10)
-
         a[self.CustomIndexIntObject(0, 1)] = 100
         assert a[0] == 100
 
     def test_setitem_obj_int(self):
         from numpypy import arange
-
         a = arange(10)
-
         a[self.CustomIntObject(1)] = 100
-
         assert a[1] == 100
 
     def test_access_swallow_exception(self):
@@ -1437,7 +1442,7 @@
         assert a[5] == 50
         b = a.imag
         assert b[7] == 0
-        raises(RuntimeError, 'b[7] = -2')
+        raises(ValueError, 'b[7] = -2')
         raises(TypeError, 'a.imag = -2')
         a = array(['abc','def'],dtype='S3')
         b = a.real
@@ -1445,7 +1450,7 @@
         assert a[1] == b[1]
         b[1] = 'xyz'
         assert a[1] == 'xyz'
-        assert a.imag[0] == 'abc'
+        assert a.imag[0] == ''
         raises(TypeError, 'a.imag = "qop"')
         a=array([[1+1j, 2-3j, 4+5j],[-6+7j, 8-9j, -2-1j]])
         assert a.real[0,1] == 2
@@ -1465,6 +1470,14 @@
         assert a[3].imag == -10
         assert a[2].imag == -5
 
+    def test_trace(self):
+        import numpypy as np
+        assert np.trace(np.eye(3)) == 3.0
+        a = np.arange(8).reshape((2,2,2))
+        assert np.array_equal(np.trace(a), [6, 8])
+        a = np.arange(24).reshape((2,2,2,3))
+        assert np.trace(a).shape == (2, 3)
+
     def test_view(self):
         from numpypy import array, int8, int16, dtype
         x = array((1, 2), dtype=int8)
@@ -1864,6 +1877,15 @@
             i2 = (i+1) * a.dtype.itemsize
             assert list(reversed(s1[i1:i2])) == s2[i1:i2]
 
+        a = array([1, -1, 10000], dtype='longfloat')
+        s1 = map(ord, a.tostring())
+        s2 = map(ord, a.byteswap().tostring())
+        assert a.dtype.itemsize >= 8
+        for i in range(a.size):
+            i1 = i * a.dtype.itemsize
+            i2 = (i+1) * a.dtype.itemsize
+            assert list(reversed(s1[i1:i2])) == s2[i1:i2]
+
     def test_clip(self):
         from numpypy import array
         a = array([1, 2, 17, -3, 12])
@@ -2148,6 +2170,7 @@
         c = b + b
         assert c.sum() == (6 + 8 + 10 + 12) * 2
         assert isinstance(c.sum(dtype='f8'), float)
+        assert isinstance(c.sum(None, 'f8'), float)
 
     def test_transpose(self):
         from numpypy import array
@@ -2463,7 +2486,7 @@
         assert type(array(True).item()) is bool
         assert type(array(3.5).item()) is float
         raises(IndexError, "array(3).item(15)")
-        raises(IndexError, "array([1, 2, 3]).item()")
+        raises(ValueError, "array([1, 2, 3]).item()")
         assert array([3]).item(0) == 3
         assert type(array([3]).item(0)) is int
         assert array([1, 2, 3]).item(-1) == 3
@@ -2638,7 +2661,7 @@
 
     def test_fromstring_types(self):
         from numpypy import (fromstring, int8, int16, int32, int64, uint8,
-            uint16, uint32, float16, float32, float64, array)
+            uint16, uint32, float16, float32, float64, longfloat, array)
         a = fromstring('\xFF', dtype=int8)
         assert a[0] == -1
         b = fromstring('\xFF', dtype=uint8)
@@ -2661,6 +2684,18 @@
         assert j[0] == 12
         k = fromstring(self.float16val, dtype=float16)
         assert k[0] == float16(5.)
+        dt =  array([5],dtype=longfloat).dtype
+        if dt.itemsize == 12:
+            from numpypy import float96
+            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00', dtype=float96)
+        elif dt.itemsize == 16:
+            from numpypy import float128
+            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00\x00\x00\x00\x00', dtype=float128)
+        elif dt.itemsize == 8:
+            skip('longfloat is float64')
+        else:
+            skip('unknown itemsize for longfloat')
+        assert m[0] == longfloat(5.)
 
     def test_fromstring_invalid(self):
         from numpypy import fromstring, uint16, uint8
@@ -2680,28 +2715,6 @@
         assert array(0, dtype='i2').tostring() == '\x00\x00'
 
 
-class AppTestRanges(BaseNumpyAppTest):
-    def test_arange(self):
-        from numpypy import arange, dtype
-        a = arange(3)
-        assert (a == [0, 1, 2]).all()
-        assert a.dtype is dtype(int)
-        a = arange(3.0)
-        assert (a == [0., 1., 2.]).all()
-        assert a.dtype is dtype(float)
-        a = arange(3, 7)
-        assert (a == [3, 4, 5, 6]).all()
-        assert a.dtype is dtype(int)
-        a = arange(3, 7, 2)
-        assert (a == [3, 5]).all()
-        a = arange(3, dtype=float)
-        assert (a == [0., 1., 2.]).all()
-        assert a.dtype is dtype(float)
-        a = arange(0, 0.8, 0.1)
-        assert len(a) == 8
-        assert arange(False, True, True).dtype is dtype(int)
-
-
 class AppTestRepr(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect:
@@ -2924,6 +2937,27 @@
         assert (a[0, 0, 0] == 500).all()
         assert a[0, 0, 0].shape == (10,)
 
+    def test_subarray_multiple_rows(self):
+        import numpypy as np
+        descr = [
+            ('x', 'i4', (2,)),
+            ('y', 'f8', (2, 2)),
+            ('z', 'u1')]
+        buf = [
+            # x     y                  z
+            ([3,2], [[6.,4.],[6.,4.]], 8),
+            ([4,3], [[7.,5.],[7.,5.]], 9),
+            ]
+        h = np.array(buf, dtype=descr)
+        assert len(h) == 2
+        skip('broken')  # XXX
+        assert np.array_equal(h['x'], np.array([buf[0][0],
+                                                buf[1][0]], dtype='i4'))
+        assert np.array_equal(h['y'], np.array([buf[0][1],
+                                                buf[1][1]], dtype='f8'))
+        assert np.array_equal(h['z'], np.array([buf[0][2],
+                                                buf[1][2]], dtype='u1'))
+
     def test_multidim_subarray(self):
         from numpypy import dtype, array
 
@@ -2998,40 +3032,3 @@
         assert x.__pypy_data__ is obj
         del x.__pypy_data__
         assert x.__pypy_data__ is None
-
-class AppTestLongDoubleDtypes(BaseNumpyAppTest):
-    def setup_class(cls):
-        from pypy.module.micronumpy import Module
-        #print dir(Module.interpleveldefs)
-        if not Module.interpleveldefs.get('longfloat', None):
-            py.test.skip('no longdouble types yet')
-        BaseNumpyAppTest.setup_class.im_func(cls)
-
-    def test_byteswap(self):
-        from numpypy import array
-
-        a = array([1, -1, 10000], dtype='longfloat')
-        s1 = map(ord, a.tostring())
-        s2 = map(ord, a.byteswap().tostring())
-        assert a.dtype.itemsize >= 8
-        for i in range(a.size):
-            i1 = i * a.dtype.itemsize
-            i2 = (i+1) * a.dtype.itemsize
-            assert list(reversed(s1[i1:i2])) == s2[i1:i2]
-
-    def test_fromstring_types(self):
-        from numpypy import (fromstring, longfloat, array)
-        dt =  array([5],dtype=longfloat).dtype
-        if dt.itemsize == 12:
-            from numpypy import float96
-            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00', dtype=float96)
-        elif dt.itemsize==16:
-            from numpypy import float128
-            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00\x00\x00\x00\x00', dtype=float128)
-        elif dt.itemsize == 8:
-            skip('longfloat is float64')
-        else:
-            skip('unknown itemsize for longfloat')
-        assert m[0] == longfloat(5.)
-
-
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -91,14 +91,14 @@
                         uncallable.add(s)
             return uncallable
         assert find_uncallable_ufuncs('int') == set()
-        assert find_uncallable_ufuncs('bool') == set()
+        assert find_uncallable_ufuncs('bool') == set(['sign'])
         assert find_uncallable_ufuncs('float') == set(
                 ['bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor',
                  'left_shift', 'right_shift', 'invert'])
         assert find_uncallable_ufuncs('complex') == set(
                 ['bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor',
                  'arctan2', 'deg2rad', 'degrees', 'rad2deg', 'radians',
-                 'fabs', 'fmod', 'invert', 'isneginf', 'isposinf',
+                 'fabs', 'fmod', 'invert', 'mod',
                  'logaddexp', 'logaddexp2', 'left_shift', 'right_shift',
                  'copysign', 'signbit', 'ceil', 'floor', 'trunc'])
 
@@ -174,7 +174,6 @@
         assert fabs(float('-inf')) == float('inf')
         assert isnan(fabs(float('nan')))
 
-
     def test_fmax(self):
         from numpypy import fmax, array
         import math
@@ -194,7 +193,6 @@
         # on Microsoft win32
         assert math.copysign(1., fmax(nnan, nan)) == math.copysign(1., nnan)
 
-
     def test_fmin(self):
         from numpypy import fmin, array
         import math
@@ -213,7 +211,6 @@
         # on Microsoft win32
         assert math.copysign(1., fmin(nnan, nan)) == math.copysign(1., nnan)
 
-
     def test_fmod(self):
         from numpypy import fmod
         import math
@@ -323,11 +320,10 @@
             reference = [0, -1, 0, 1, 0]
             if dtype[0] == 'u':
                 reference[1] = 0
-            # XXX need to fix specialization issue in types.py first
-            #elif dtype == 'int32':
-            #        reference[2] = -2147483648
-            #elif dtype == 'int64':
-            #        reference[2] = -9223372036854775808
+            elif dtype == 'int32':
+                    reference[2] = -2147483648
+            elif dtype == 'int64':
+                    reference[2] = -9223372036854775808
             a = array([-2, -1, 0, 1, 2], dtype)
             b = reciprocal(a)
             assert (b == reference).all()
@@ -369,7 +365,6 @@
         c = array([10.5+11.5j, -15.2-100.3456j, 0.2343+11.123456j])
         assert (c.round(0) == [10.+12.j, -15-100j, 0+11j]).all()
 
-
     def test_copysign(self):
         from numpypy import array, copysign
 
@@ -437,7 +432,6 @@
 
         assert expm1(1e-50) == 1e-50
 
-
     def test_sin(self):
         import math
         from numpypy import array, sin
@@ -705,6 +699,8 @@
         assert (~a == [-2, -3, -4, -5]).all()
         assert (bitwise_not(a) == ~a).all()
         assert (invert(a) == ~a).all()
+        assert invert(True) == False
+        assert invert(False) == True
 
     def test_shift(self):
         from numpypy import left_shift, right_shift, bool
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -12,7 +12,7 @@
 from rpython.rlib.rawstorage import (alloc_raw_storage, raw_storage_setitem,
                                   raw_storage_getitem)
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import widen, byteswap, r_ulonglong
+from rpython.rlib.rarithmetic import widen, byteswap, r_ulonglong, most_neg_value_of
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rstruct.runpack import runpack
 from rpython.rlib.rstruct.nativefmttable import native_is_bigendian
@@ -248,14 +248,6 @@
     def isinf(self, v):
         return False
 
-    @raw_unary_op
-    def isneginf(self, v):
-        return False
-
-    @raw_unary_op
-    def isposinf(self, v):
-        return False
-
     @raw_binary_op
     def eq(self, v1, v2):
         return v1 == v2
@@ -401,7 +393,7 @@
 
     @simple_unary_op
     def invert(self, v):
-        return ~v
+        return not v
 
     @raw_unary_op
     def isfinite(self, v):
@@ -497,14 +489,6 @@
     def isinf(self, v):
         return False
 
-    @raw_unary_op
-    def isposinf(self, v):
-        return False
-
-    @raw_unary_op
-    def isneginf(self, v):
-        return False
-
     @simple_binary_op
     def bitwise_and(self, v1, v2):
         return v1 & v2
@@ -521,18 +505,17 @@
     def invert(self, v):
         return ~v
 
-    @simple_unary_op
+    @specialize.argtype(1)
     def reciprocal(self, v):
-        if v == 0:
+        raw = self.for_computation(self.unbox(v))
+        ans = 0
+        if raw == 0:
             # XXX good place to warn
-            # XXX can't do the following, func is specialized only on argtype(v)
-            # (which is the same for all int classes)
-            #if self.T in (rffi.INT, rffi.LONG):
-            #    return most_neg_value_of(self.T)
-            return 0
-        if abs(v) == 1:
-            return v
-        return 0
+            if self.T is rffi.INT or self.T is rffi.LONG:
+                ans = most_neg_value_of(self.T)
+        elif abs(raw) == 1:
+            ans = raw
+        return self.box(ans)
 
     @specialize.argtype(1)
     def round(self, v, decimals=0):
@@ -948,14 +931,6 @@
         return rfloat.isinf(v)
 
     @raw_unary_op
-    def isneginf(self, v):
-        return rfloat.isinf(v) and v < 0
-
-    @raw_unary_op
-    def isposinf(self, v):
-        return rfloat.isinf(v) and v > 0
-
-    @raw_unary_op
     def isfinite(self, v):
         return not (rfloat.isinf(v) or rfloat.isnan(v))
 
@@ -1148,9 +1123,13 @@
         return v
 
     def to_builtin_type(self, space, box):
-        real,imag = self.for_computation(self.unbox(box))
+        real, imag = self.for_computation(self.unbox(box))
         return space.newcomplex(real, imag)
 
+    def bool(self, v):
+        real, imag = self.for_computation(self.unbox(v))
+        return bool(real) or bool(imag)
+
     def read_bool(self, arr, i, offset):
         v = self.for_computation(self._read(arr.storage, i, offset))
         return bool(v[0]) or bool(v[1])
@@ -1661,16 +1640,21 @@
 
 NonNativeComplex128 = Complex128
 
-if interp_boxes.ENABLED_LONG_DOUBLE and interp_boxes.long_double_size == 12:
-    class Float96(BaseType, Float):
+if interp_boxes.long_double_size == 8:
+    FloatLong = Float64
+    NonNativeFloatLong = NonNativeFloat64
+    ComplexLong = Complex128
+    NonNativeComplexLong = NonNativeComplex128
+
+elif interp_boxes.long_double_size in (12, 16):
+    class FloatLong(BaseType, Float):
         _attrs_ = ()
 
         T = rffi.LONGDOUBLE
-        BoxType = interp_boxes.W_Float96Box
-        format_code = "q"
+        BoxType = interp_boxes.W_FloatLongBox
 
         def runpack_str(self, s):
-            assert len(s) == 12
+            assert len(s) == interp_boxes.long_double_size
             fval = unpack_float80(s, native_is_bigendian)
             return self.box(fval)
 
@@ -1680,46 +1664,17 @@
             pack_float80(result, value, 10, not native_is_bigendian)
             return self.box(unpack_float80(result.build(), native_is_bigendian))
 
-    NonNativeFloat96 = Float96
+    NonNativeFloatLong = FloatLong
 
-    class Complex192(ComplexFloating, BaseType):
+    class ComplexLong(ComplexFloating, BaseType):
         _attrs_ = ()
 
         T = rffi.LONGDOUBLE
-        BoxType = interp_boxes.W_Complex192Box
-        ComponentBoxType = interp_boxes.W_Float96Box
+        BoxType = interp_boxes.W_ComplexLongBox
+        ComponentBoxType = interp_boxes.W_FloatLongBox
 
-    NonNativeComplex192 = Complex192
+    NonNativeComplexLong = ComplexLong
 
-elif interp_boxes.ENABLED_LONG_DOUBLE and interp_boxes.long_double_size == 16:
-    class Float128(BaseType, Float):
-        _attrs_ = ()
-
-        T = rffi.LONGDOUBLE
-        BoxType = interp_boxes.W_Float128Box
-        format_code = "q"
-
-        def runpack_str(self, s):
-            assert len(s) == 16
-            fval = unpack_float80(s, native_is_bigendian)
-            return self.box(fval)
-
-        def byteswap(self, w_v):
-            value = self.unbox(w_v)
-            result = StringBuilder(10)
-            pack_float80(result, value, 10, not native_is_bigendian)
-            return self.box(unpack_float80(result.build(), native_is_bigendian))
-
-    NonNativeFloat128 = Float128
-
-    class Complex256(ComplexFloating, BaseType):
-        _attrs_ = ()
-
-        T = rffi.LONGDOUBLE
-        BoxType = interp_boxes.W_Complex256Box
-        ComponentBoxType = interp_boxes.W_Float128Box
-
-    NonNativeComplex256 = Complex256
 
 class BaseStringType(object):
     _mixin_ = True
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -245,7 +245,7 @@
         guard_not_invalidated(descr=...)
         p52 = call(ConstClass(str_decode_ascii__raise_unicode_exception_decode), ConstPtr(ptr38), 3, 1, descr=<Callr . rii EF=4>)
         guard_no_exception(descr=...)
-        p53 = getfield_gc_pure(p52, descr=<FieldP tuple2.item0 8>)
+        p53 = getfield_gc_pure(p52, descr=<FieldP tuple2.item0 .>)
         guard_nonnull(p53, descr=...)
         --TICK--
         jump(..., descr=...)
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -19,20 +19,6 @@
 
 RPY_LOCK_FAILURE, RPY_LOCK_ACQUIRED, RPY_LOCK_INTR = range(3)
 
-##import sys
-##def debug(msg, n):
-##    return

From noreply at buildbot.pypy.org  Tue Oct 22 11:00:50 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Oct 2013 11:00:50 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: undo the not-break-in-loop-body
Message-ID: <20131022090050.C548D1C30A6@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67487:51b6c398a0ff
Date: 2013-10-18 15:07 +0200
http://bitbucket.org/pypy/pypy/changeset/51b6c398a0ff/

Log:	undo the not-break-in-loop-body

diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -43,14 +43,6 @@
            }
 
     def rewrite(self, operations):
-        # try to find a loop body:
-        last_label = None
-        in_loop_body = False
-        if operations[-1].getopnum() == rop.JUMP:
-            for op in operations:
-                if op.getopnum() == rop.LABEL:
-                    last_label = op
-        
         # overridden method from parent class
         #
         insert_transaction_break = False
@@ -131,10 +123,9 @@
                 self.emitting_an_operation_that_can_collect()
                 self.next_op_may_be_in_new_transaction()
 
-                if (not in_loop_body and (
-                        op.getopnum() == rop.CALL_MAY_FORCE or
-                        op.getopnum() == rop.CALL_ASSEMBLER or
-                        op.getopnum() == rop.CALL_RELEASE_GIL)):
+                if (op.getopnum() == rop.CALL_MAY_FORCE or
+                    op.getopnum() == rop.CALL_ASSEMBLER or
+                    op.getopnum() == rop.CALL_RELEASE_GIL):
                     # insert more transaction breaks after function
                     # calls since they are likely to return as
                     # inevitable transactions
@@ -174,8 +165,6 @@
                 self.known_lengths.clear()
                 self.always_inevitable = False
                 self.newops.append(op)
-                if op is last_label:
-                    in_loop_body = True
                 continue
             # ----------  jumps  ----------
             if op.getopnum() == rop.JUMP:
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -1185,6 +1185,8 @@
         """, calldescr2=calldescr2)
 
     def test_no_transactionbreak_in_loop_body(self):
+        py.test.skip("actually not good")
+
         class fakeextrainfo:
             def call_needs_inevitable(self):
                 return False

From noreply at buildbot.pypy.org  Tue Oct 22 11:00:48 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Oct 2013 11:00:48 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: fix little mistake in stmrewrite and
 make it not emit transaction breaks in loops (except before JUMP)
Message-ID: <20131022090048.96DE31C3085@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67486:f979ce4ca940
Date: 2013-10-18 13:42 +0200
http://bitbucket.org/pypy/pypy/changeset/f979ce4ca940/

Log:	fix little mistake in stmrewrite and make it not emit transaction
	breaks in loops (except before JUMP)

diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -43,6 +43,14 @@
            }
 
     def rewrite(self, operations):
+        # try to find a loop body:
+        last_label = None
+        in_loop_body = False
+        if operations[-1].getopnum() == rop.JUMP:
+            for op in operations:
+                if op.getopnum() == rop.LABEL:
+                    last_label = op
+        
         # overridden method from parent class
         #
         insert_transaction_break = False
@@ -93,6 +101,7 @@
                         ResOperation(rop.STM_TRANSACTION_BREAK, [], None))
                     insert_transaction_break = False
                     self.emitting_an_operation_that_can_collect()
+                    self.next_op_may_be_in_new_transaction()
                 else:
                     assert insert_transaction_break is False
 
@@ -120,9 +129,12 @@
             # ----------  calls  ----------
             if op.is_call():
                 self.emitting_an_operation_that_can_collect()
-                if (op.getopnum() == rop.CALL_MAY_FORCE or
-                    op.getopnum() == rop.CALL_ASSEMBLER or
-                    op.getopnum() == rop.CALL_RELEASE_GIL):
+                self.next_op_may_be_in_new_transaction()
+
+                if (not in_loop_body and (
+                        op.getopnum() == rop.CALL_MAY_FORCE or
+                        op.getopnum() == rop.CALL_ASSEMBLER or
+                        op.getopnum() == rop.CALL_RELEASE_GIL)):
                     # insert more transaction breaks after function
                     # calls since they are likely to return as
                     # inevitable transactions
@@ -157,9 +169,13 @@
             # ----------  labels  ----------
             if op.getopnum() == rop.LABEL:
                 self.emitting_an_operation_that_can_collect()
+                self.next_op_may_be_in_new_transaction()
+                
                 self.known_lengths.clear()
                 self.always_inevitable = False
                 self.newops.append(op)
+                if op is last_label:
+                    in_loop_body = True
                 continue
             # ----------  jumps  ----------
             if op.getopnum() == rop.JUMP:
@@ -188,10 +204,9 @@
         assert not insert_transaction_break
         return self.newops
 
-    def emitting_an_operation_that_can_collect(self):
-        GcRewriterAssembler.emitting_an_operation_that_can_collect(self)
+    def next_op_may_be_in_new_transaction(self):
         self.known_category.clear()
-
+        
     def write_to_read_categories(self):
         for v, c in self.known_category.items():
             if c == 'W':
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -227,9 +227,9 @@
             [p1, p3]
             cond_call_stm_b(p3, descr=P2Wdescr)
             setfield_gc(p3, p1, descr=tzdescr)
-            p2 = call_malloc_gc(ConstClass(malloc_big_fixedsize),    \
-                                %(tdescr.size)d, %(tdescr.tid)d, \
-                                descr=malloc_big_fixedsize_descr)
+            p2 = call_malloc_nursery(%(tdescr.size)d)
+            setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr)
+            stm_set_revision_gc(p2, descr=revdescr)
             cond_call_stm_b(p3, descr=P2Wdescr)
             setfield_gc(p3, p1, descr=tzdescr)
             stm_transaction_break()
@@ -247,9 +247,9 @@
             [p1]
             cond_call_stm_b(p1, descr=P2Rdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
-            p3 = call_malloc_gc(ConstClass(malloc_big_fixedsize),    \
-                                %(tdescr.size)d, %(tdescr.tid)d, \
-                                descr=malloc_big_fixedsize_descr)
+            p3 = call_malloc_nursery(%(tdescr.size)d)
+            setfield_gc(p3, %(tdescr.tid)d, descr=tiddescr)
+            stm_set_revision_gc(p3, descr=revdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
             stm_transaction_break()
             jump(p2)
@@ -263,9 +263,9 @@
             jump(p2)
         """, """
             [p1]
-            p2 = call_malloc_gc(ConstClass(malloc_big_fixedsize),    \
-                                %(tdescr.size)d, %(tdescr.tid)d, \
-                                descr=malloc_big_fixedsize_descr)
+            p2 = call_malloc_nursery(%(tdescr.size)d)
+            setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr)
+            stm_set_revision_gc(p2, descr=revdescr)
             setfield_gc(p2, p1, descr=tzdescr)
             stm_transaction_break()
             jump(p2)
@@ -481,9 +481,9 @@
             jump(p2)
         """, """
             [p0]
-            p1 = call_malloc_gc(ConstClass(malloc_big_fixedsize),    \
-                                %(tdescr.size)d, %(tdescr.tid)d, \
-                                descr=malloc_big_fixedsize_descr)
+            p1 = call_malloc_nursery(%(tdescr.size)d)
+            setfield_gc(p1, %(tdescr.tid)d, descr=tiddescr)
+            stm_set_revision_gc(p1, descr=revdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
             stm_transaction_break()
             jump(p2)
@@ -1183,6 +1183,45 @@
             stm_set_revision_gc(p2, descr=revdescr)
             setfield_gc(p2, 5, descr=blendescr)
         """, calldescr2=calldescr2)
+
+    def test_no_transactionbreak_in_loop_body(self):
+        class fakeextrainfo:
+            def call_needs_inevitable(self):
+                return False
+        T = rffi.CArrayPtr(rffi.TIME_T)
+        calldescr2 = get_call_descr(self.gc_ll_descr, [T], rffi.TIME_T,
+                                    fakeextrainfo())
+
+        self.check_rewrite("""
+            []
+            call_may_force(123, descr=calldescr2)
+            guard_not_forced() []
+            
+            label()
+            
+            call_may_force(123, descr=calldescr2)
+            guard_not_forced() []
+
+            i0 = int_add(1, 2)
+            
+            jump()
+        """, """
+            []
+            call_may_force(123, descr=calldescr2)
+            guard_not_forced() []
+            stm_transaction_break()
+            
+            label()
+            
+            call_may_force(123, descr=calldescr2)
+            guard_not_forced() []
+
+            i0 = int_add(1, 2)
+            
+            stm_transaction_break()
+            jump()
+        """, calldescr2=calldescr2)
+
         
 
 

From noreply at buildbot.pypy.org  Tue Oct 22 11:00:52 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Oct 2013 11:00:52 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: update TODO
Message-ID: <20131022090052.050A21C3085@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67488:e7c18c48620d
Date: 2013-10-18 15:35 +0200
http://bitbucket.org/pypy/pypy/changeset/e7c18c48620d/

Log:	update TODO

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -52,8 +52,6 @@
 JIT
 ~~~
 
-* reimplement the fast-path of the nursery allocations in the GC
-** use this for frame allocation in stmrewrite for call_assembler
 * use specialized barriers in JIT
 * optimize produced assembler code
 * avoid calling aroundstate.after() for call_release_gil and instead
@@ -61,4 +59,8 @@
 * maybe GUARD_NOT_INEVITABLE after call_may_force, call_assembler
   which is a small check if we are inevitable and does a transaction_break
   if we are.
+** do not access thread-locals through thread_descriptor, but directly
+** have two versions of stm_transaction_break(1/2). One for after calls
+   which simply checks if the transaction is inevitable, and one to place
+   before JUMPs which calls stm_should_break_transaction()
 * look at XXXs for STM everywhere

From noreply at buildbot.pypy.org  Tue Oct 22 11:00:53 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Oct 2013 11:00:53 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: import stmgc with new thread-locals
 (needs next commit to work)
Message-ID: <20131022090053.408471C3085@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67489:dea35e8a8a38
Date: 2013-10-21 14:15 +0200
http://bitbucket.org/pypy/pypy/changeset/dea35e8a8a38/

Log:	import stmgc with new thread-locals (needs next commit to work)

diff --git a/rpython/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c
--- a/rpython/translator/stm/src_stm/et.c
+++ b/rpython/translator/stm/src_stm/et.c
@@ -57,7 +57,7 @@
 }
 
 
-
+__thread int stm_active;
 __thread struct tx_descriptor *thread_descriptor = NULL;
 
 /* 'global_cur_time' is normally a multiple of 2, except when we turn
@@ -104,8 +104,8 @@
 {
   /* Assert that we are running a transaction.
    *      Returns True if this transaction is inevitable. */
-  assert(d->active == 1 + !d->setjmp_buf);
-  return d->active == 2;
+  assert(*d->active_ref == 1 + !d->setjmp_buf);
+  return *d->active_ref == 2;
 }
 
 static pthread_mutex_t mutex_inevitable = PTHREAD_MUTEX_INITIALIZER;
@@ -121,7 +121,7 @@
   pthread_mutex_lock(&mutex_inevitable);
   stm_start_sharedlock();
 
-  if (d->active < 0)
+  if (*d->active_ref < 0)
     {
       inev_mutex_release();
       AbortNowIfDelayed();
@@ -706,7 +706,7 @@
     }
 
   struct tx_descriptor *d = thread_descriptor;
-  assert(d->active >= 1);
+  assert(*d->active_ref >= 1);
 
   /* We need the collection_lock for the sequel; this is required notably
      because we're about to edit flags on a protected object.
@@ -890,7 +890,7 @@
 void SpinLoop(int num)
 {
   struct tx_descriptor *d = thread_descriptor;
-  assert(d->active >= 1);
+  assert(*d->active_ref >= 1);
   assert(num < SPINLOOP_REASONS);
   d->num_spinloops[num]++;
   smp_spinloop();
@@ -925,7 +925,7 @@
       assert(!stm_has_got_any_lock(d));
     }
 
-  assert(d->active != 0);
+  assert(*d->active_ref != 0);
   assert(!is_inevitable(d));
   assert(num < ABORT_REASONS);
   d->num_aborts[num]++;
@@ -990,7 +990,7 @@
   SpinLoop(SPLP_ABORT);
 
   /* make the transaction no longer active */
-  d->active = 0;
+  *d->active_ref = 0;
   d->atomic = 0;
 
   /* release the lock */
@@ -1032,22 +1032,22 @@
 
 void AbortTransactionAfterCollect(struct tx_descriptor *d, int reason)
 {
-  if (d->active >= 0)
+  if (*d->active_ref >= 0)
     {
       dprintf(("abort %d after collect!\n", reason));
-      assert(d->active == 1);   /* not 2, which means inevitable */
-      d->active = -reason;
+      assert(*d->active_ref == 1);   /* not 2, which means inevitable */
+      *d->active_ref = -reason;
     }
-  assert(d->active < 0);
+  assert(*d->active_ref < 0);
 }
 
 void AbortNowIfDelayed(void)
 {
   struct tx_descriptor *d = thread_descriptor;
-  if (d->active < 0)
+  if (*d->active_ref < 0)
     {
-      int reason = -d->active;
-      d->active = 1;
+      int reason = -*d->active_ref;
+      *d->active_ref = 1;
       AbortTransaction(reason);
     }
 }
@@ -1075,9 +1075,9 @@
 static void init_transaction(struct tx_descriptor *d)
 {
   assert(d->atomic == 0);
-  assert(d->active == 0);
+  assert(*d->active_ref == 0);
   stm_start_sharedlock();
-  assert(d->active == 0);
+  assert(*d->active_ref == 0);
 
   if (clock_gettime(CLOCK_MONOTONIC, &d->start_real_time) < 0) {
     d->start_real_time.tv_nsec = -1;
@@ -1098,7 +1098,7 @@
 {
   struct tx_descriptor *d = thread_descriptor;
   init_transaction(d);
-  d->active = 1;
+  *d->active_ref = 1;
   d->setjmp_buf = buf;
   d->longjmp_callback = longjmp_callback;
   d->old_thread_local_obj = stm_thread_local_obj;
@@ -1430,7 +1430,7 @@
 {   /* must save roots around this call */
   revision_t cur_time;
   struct tx_descriptor *d = thread_descriptor;
-  assert(d->active >= 1);
+  assert(*d->active_ref >= 1);
   assert(d->atomic == 0);
   dprintf(("CommitTransaction(%p)\n", d));
   spinlock_acquire(d->public_descriptor->collection_lock, 'C');  /*committing*/
@@ -1503,7 +1503,7 @@
 
   spinlock_release(d->public_descriptor->collection_lock);
   d->num_commits++;
-  d->active = 0;
+  *d->active_ref = 0;
   stm_stop_sharedlock();
 
   /* clear the list of callbacks that would have been called
@@ -1517,7 +1517,7 @@
 {
   d->setjmp_buf = NULL;
   d->old_thread_local_obj = NULL;
-  d->active = 2;
+  *d->active_ref = 2;
   d->reads_size_limit_nonatomic = 0;
   update_reads_size_limit(d);
   dprintf(("make_inevitable(%p)\n", d));
@@ -1544,7 +1544,7 @@
 {   /* must save roots around this call */
   revision_t cur_time;
   struct tx_descriptor *d = thread_descriptor;
-  if (d == NULL || d->active != 1)
+  if (d == NULL || *d->active_ref != 1)
     return;  /* I am already inevitable, or not in a transaction at all
                 (XXX statically we should know when we're outside
                 a transaction) */
@@ -1743,6 +1743,9 @@
       assert(d->my_lock & 1);
       assert(d->my_lock >= LOCKED);
       stm_private_rev_num = -d->my_lock;
+      d->active_ref = &stm_active;
+      d->nursery_current_ref = &stm_nursery_current;
+      d->nursery_nextlimit_ref = &stm_nursery_nextlimit;
       d->private_revision_ref = &stm_private_rev_num;
       d->read_barrier_cache_ref = &stm_read_barrier_cache;
       stm_thread_local_obj = NULL;
@@ -1769,7 +1772,7 @@
     revision_t i;
     struct tx_descriptor *d = thread_descriptor;
     assert(d != NULL);
-    assert(d->active == 0);
+    assert(*d->active_ref == 0);
 
     /* our nursery is empty at this point.  The list 'stolen_objects'
        should have been emptied at the previous minor collection and
diff --git a/rpython/translator/stm/src_stm/et.h b/rpython/translator/stm/src_stm/et.h
--- a/rpython/translator/stm/src_stm/et.h
+++ b/rpython/translator/stm/src_stm/et.h
@@ -175,8 +175,8 @@
   unsigned long count_reads;
   unsigned long reads_size_limit;        /* see should_break_tr. */
   unsigned long reads_size_limit_nonatomic;
-  int active;    /* 0 = inactive, 1 = regular, 2 = inevitable,
-                    negative = killed by collection */
+  int *active_ref;    /* 0 = inactive, 1 = regular, 2 = inevitable,
+                         negative = killed by collection */
   struct timespec start_real_time;
   int max_aborts;
   unsigned int num_commits;
@@ -201,6 +201,7 @@
 
 extern __thread struct tx_descriptor *thread_descriptor;
 extern __thread revision_t stm_private_rev_num;
+extern __thread int stm_active;
 extern struct tx_public_descriptor *stm_descriptor_array[];
 extern struct tx_descriptor *stm_tx_head;
 
diff --git a/rpython/translator/stm/src_stm/extra.c b/rpython/translator/stm/src_stm/extra.c
--- a/rpython/translator/stm/src_stm/extra.c
+++ b/rpython/translator/stm/src_stm/extra.c
@@ -26,7 +26,7 @@
 void stm_call_on_abort(void *key, void callback(void *))
 {
     struct tx_descriptor *d = thread_descriptor;
-    if (d == NULL || d->active != 1)
+    if (d == NULL || *d->active_ref != 1)
         return;   /* ignore callbacks if we're outside a transaction or
                      in an inevitable transaction (which cannot abort) */
     if (callback == NULL) {
@@ -49,7 +49,7 @@
 void stm_invoke_callbacks_on_abort(struct tx_descriptor *d)
 {
     wlog_t *item;
-    assert(d->active == 0);
+    assert(*d->active_ref == 0);
 
     G2L_LOOP_FORWARD(d->callbacks_on_abort, item) {
         void *key = (void *)item->addr;
@@ -287,7 +287,7 @@
         output->signature_packed = 127;
         output->elapsed_time = elapsed_time;
         output->abort_reason = abort_reason;
-        output->active = d->active;
+        output->active = *d->active_ref;
         output->atomic = d->atomic;
         output->count_reads = d->count_reads;
         output->reads_size_limit_nonatomic = d->reads_size_limit_nonatomic;
diff --git a/rpython/translator/stm/src_stm/gcpage.c b/rpython/translator/stm/src_stm/gcpage.c
--- a/rpython/translator/stm/src_stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/gcpage.c
@@ -650,7 +650,7 @@
     /* If we're aborting this transaction anyway, we don't need to do
      * more here.
      */
-    if (d->active < 0) {
+    if (*d->active_ref < 0) {
         /* already "aborted" during forced minor collection
            clear list of read objects so that a possible minor collection 
            before the abort doesn't trip 
@@ -660,7 +660,7 @@
         return;
     }
 
-    if (d->active == 2) {
+    if (*d->active_ref == 2) {
         /* inevitable transaction: clear the list of read objects */
         gcptrlist_clear(&d->list_of_read_objects);
     }
diff --git a/rpython/translator/stm/src_stm/nursery.c b/rpython/translator/stm/src_stm/nursery.c
--- a/rpython/translator/stm/src_stm/nursery.c
+++ b/rpython/translator/stm/src_stm/nursery.c
@@ -1,6 +1,10 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 #include "stmimpl.h"
 
+
+__thread char *stm_nursery_current;
+__thread char *stm_nursery_nextlimit;
+
 int stmgc_is_in_nursery(struct tx_descriptor *d, gcptr obj)
 {
     return (d->nursery_base <= (char*)obj && ((char*)obj) < d->nursery_end);
@@ -33,8 +37,8 @@
     assert(d->nursery_base == NULL);
     d->nursery_base = stm_malloc(GC_NURSERY);       /* start of nursery */
     d->nursery_end = d->nursery_base + GC_NURSERY;  /* end of nursery */
-    d->nursery_current = d->nursery_base;           /* current position */
-    d->nursery_nextlimit = d->nursery_base;         /* next section limit */
+    *d->nursery_current_ref = d->nursery_base;           /* current position */
+    *d->nursery_nextlimit_ref = d->nursery_base;         /* next section limit */
     d->nursery_cleared = NC_REGULAR;
 
     dprintf(("minor: nursery is at [%p to %p]\n", d->nursery_base,
@@ -49,7 +53,7 @@
        this assert (committransaction() -> 
        updatechainheads() -> stub_malloc() -> ...): */
     assert(!minor_collect_anything_to_do(d)
-           || d->nursery_current == d->nursery_end);
+           || *d->nursery_current_ref == d->nursery_end);
     stm_free(d->nursery_base);
 
     gcptrlist_delete(&d->old_objects_to_trace);
@@ -60,7 +64,7 @@
 void stmgc_minor_collect_soon(void)
 {
     struct tx_descriptor *d = thread_descriptor;
-    d->nursery_current = d->nursery_end;
+    *d->nursery_current_ref = d->nursery_end;
 }
 
 inline static gcptr allocate_nursery(size_t size, revision_t tid)
@@ -68,11 +72,11 @@
     /* if 'tid == -1', we must not collect */
     struct tx_descriptor *d = thread_descriptor;
     gcptr P;
-    char *cur = d->nursery_current;
+    char *cur = *d->nursery_current_ref;
     char *end = cur + size;
     assert((size & 3) == 0);
-    d->nursery_current = end;
-    if (end > d->nursery_nextlimit) {
+    *d->nursery_current_ref = end;
+    if (end > *d->nursery_nextlimit_ref) {
         P = allocate_next_section(size, tid);
     }
     else {
@@ -95,7 +99,7 @@
 {
     /* XXX inline the fast path */
     assert(tid == (tid & STM_USER_TID_MASK));
-    assert(thread_descriptor->active > 0);
+    assert(*thread_descriptor->active_ref > 0);
     gcptr P = allocate_nursery(size, tid);
     P->h_revision = stm_private_rev_num;
     assert(P->h_original == 0);  /* null-initialized already */
@@ -403,7 +407,7 @@
     gcptr *items = d->list_of_read_objects.items;
     assert(d->list_of_read_objects.size >= limit);
 
-    if (d->active == 2) {
+    if (*d->active_ref == 2) {
         /* inevitable transaction: clear the list of read objects */
         gcptrlist_clear(&d->list_of_read_objects);
     }
@@ -502,10 +506,10 @@
        Second, if the thread is really idle, then its nursery is sent
        back to the system until it's really needed.
     */
-    if ((d->nursery_nextlimit - d->nursery_base) < GC_NURSERY / 10) {
+    if ((*d->nursery_nextlimit_ref - d->nursery_base) < GC_NURSERY / 10) {
         size_t already_cleared = 0;
         if (d->nursery_cleared == NC_ALREADY_CLEARED) {
-            already_cleared = d->nursery_end - d->nursery_current;
+            already_cleared = d->nursery_end - *d->nursery_current_ref;
         }
         stm_clear_large_memory_chunk(d->nursery_base, GC_NURSERY,
                                      already_cleared);
@@ -514,7 +518,7 @@
     else {
         d->nursery_cleared = NC_REGULAR;
 #if defined(_GC_DEBUG)
-        memset(d->nursery_current, 0xEE, d->nursery_end - d->nursery_current);
+        memset(*d->nursery_current_ref, 0xEE, d->nursery_end - *d->nursery_current_ref);
 #endif
     }
 
@@ -532,8 +536,8 @@
     if (d->nursery_cleared == NC_ALREADY_CLEARED)
         memset(d->nursery_base, 0, GC_NURSERY);
 #endif
-    d->nursery_current = d->nursery_base;
-    d->nursery_nextlimit = d->nursery_base;
+    *d->nursery_current_ref = d->nursery_base;
+    *d->nursery_nextlimit_ref = d->nursery_base;
 
     assert(!minor_collect_anything_to_do(d));
 }
@@ -541,7 +545,7 @@
 void stmgc_minor_collect(void)
 {
     struct tx_descriptor *d = thread_descriptor;
-    assert(d->active >= 1);
+    assert(*d->active_ref >= 1);
     minor_collect(d);
     AbortNowIfDelayed();
 }
@@ -555,7 +559,7 @@
 #ifndef NDEBUG
 int minor_collect_anything_to_do(struct tx_descriptor *d)
 {
-    if (d->nursery_current == d->nursery_base /*&&
+    if (*d->nursery_current_ref == d->nursery_base /*&&
         !g2l_any_entry(&d->young_objects_outside_nursery)*/ ) {
         /* there is no young object */
         assert(gcptrlist_size(&d->public_with_young_copy) == 0);
@@ -589,7 +593,7 @@
        First fix 'nursery_current', left to a bogus value by the caller.
     */
     struct tx_descriptor *d = thread_descriptor;
-    d->nursery_current -= allocate_size;
+    *d->nursery_current_ref -= allocate_size;
 
     /* Are we asking for a "reasonable" number of bytes, i.e. a value
        at most equal to one section?
@@ -609,8 +613,8 @@
     }
 
     /* Are we at the end of the nursery? */
-    if (d->nursery_nextlimit == d->nursery_end ||
-        d->nursery_current == d->nursery_end) {   // stmgc_minor_collect_soon()
+    if (*d->nursery_nextlimit_ref == d->nursery_end ||
+        *d->nursery_current_ref == d->nursery_end) {   // stmgc_minor_collect_soon()
         /* Yes */
         if (tid == -1)
             return NULL;    /* cannot collect */
@@ -620,19 +624,19 @@
         stmgc_minor_collect();
         stmgcpage_possibly_major_collect(0);
 
-        assert(d->nursery_current == d->nursery_base);
-        assert(d->nursery_nextlimit == d->nursery_base);
+        assert(*d->nursery_current_ref == d->nursery_base);
+        assert(*d->nursery_nextlimit_ref == d->nursery_base);
     }
 
     /* Clear the next section */
     if (d->nursery_cleared != NC_ALREADY_CLEARED)
-        memset(d->nursery_nextlimit, 0, GC_NURSERY_SECTION);
-    d->nursery_nextlimit += GC_NURSERY_SECTION;
+        memset(*d->nursery_nextlimit_ref, 0, GC_NURSERY_SECTION);
+    *d->nursery_nextlimit_ref += GC_NURSERY_SECTION;
 
     /* Return the object from there */
-    gcptr P = (gcptr)d->nursery_current;
-    d->nursery_current += allocate_size;
-    assert(d->nursery_current <= d->nursery_nextlimit);
+    gcptr P = (gcptr)*d->nursery_current_ref;
+    *d->nursery_current_ref += allocate_size;
+    assert(*d->nursery_current_ref <= *d->nursery_nextlimit_ref);
 
     P->h_tid = tid;
     assert_cleared(((char *)P) + sizeof(revision_t),
diff --git a/rpython/translator/stm/src_stm/nursery.h b/rpython/translator/stm/src_stm/nursery.h
--- a/rpython/translator/stm/src_stm/nursery.h
+++ b/rpython/translator/stm/src_stm/nursery.h
@@ -25,8 +25,8 @@
 
 #define NURSERY_FIELDS_DECL                                             \
     /* the nursery */                                                   \
-    char *nursery_current;                                              \
-    char *nursery_nextlimit;                                            \
+    char **nursery_current_ref;                                         \
+    char **nursery_nextlimit_ref;                                       \
     char *nursery_end;                                                  \
     char *nursery_base;                                                 \
     enum { NC_REGULAR, NC_ALREADY_CLEARED } nursery_cleared;            \
@@ -58,6 +58,9 @@
 
 
 struct tx_descriptor;  /* from et.h */
+extern __thread char *stm_nursery_current;
+extern __thread char *stm_nursery_nextlimit;
+
 
 void stmgc_init_nursery(void);
 void stmgc_done_nursery(void);
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-3acc863a00a8
+2c23968e3d8f
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -197,12 +197,14 @@
 /* macro functionality */
 
 extern __thread gcptr *stm_shadowstack;
+extern __thread int stm_active;
+extern __thread char *stm_nursery_current;
+extern __thread char *stm_nursery_nextlimit;
 
 #define stm_push_root(obj)  (*stm_shadowstack++ = (obj))
 #define stm_pop_root()      (*--stm_shadowstack)
 
 extern __thread revision_t stm_private_rev_num;
-extern __thread struct tx_descriptor *thread_descriptor; /* XXX: stm_ prefix */
 gcptr stm_DirectReadBarrier(gcptr);
 gcptr stm_WriteBarrier(gcptr);
 gcptr stm_RepeatReadBarrier(gcptr);
diff --git a/rpython/translator/stm/src_stm/stmsync.c b/rpython/translator/stm/src_stm/stmsync.c
--- a/rpython/translator/stm/src_stm/stmsync.c
+++ b/rpython/translator/stm/src_stm/stmsync.c
@@ -43,7 +43,7 @@
                                    d->reads_size_limit_nonatomic));
     /* if is_inevitable(), reads_size_limit_nonatomic should be 0
        (and thus reads_size_limit too, if !d->atomic.) */
-    if (d->active == 2)
+    if (*d->active_ref == 2)
         assert(d->reads_size_limit_nonatomic == 0);
 #endif
 
@@ -168,7 +168,7 @@
            has configured 'reads_size_limit_nonatomic' to a smaller value.
            When such a shortened transaction succeeds, the next one will
            see its length limit doubled, up to the maximum. */
-        if (counter == 0 && d->active != 2) {
+        if (counter == 0 && *d->active_ref != 2) {
             unsigned long limit = d->reads_size_limit_nonatomic;
             if (limit != 0 && limit < (stm_regular_length_limit >> 1))
                 limit = (limit << 1) | 1;
@@ -183,7 +183,7 @@
             /* atomic transaction: a common case is that callback() returned
                even though we are atomic because we need a major GC.  For
                that case, release and reaquire the rw lock here. */
-            assert(d->active >= 1);
+            assert(*d->active_ref >= 1);
             stm_possible_safe_point();
         }
 
@@ -218,7 +218,7 @@
 {   /* must save roots around this call */
     struct tx_descriptor *d = thread_descriptor;
     if (d->atomic) {
-        assert(d->active >= 1);
+        assert(*d->active_ref >= 1);
         stm_possible_safe_point();
     }
     else {
@@ -267,7 +267,7 @@
 int stm_in_transaction(void)
 {
     struct tx_descriptor *d = thread_descriptor;
-    return d && d->active;
+    return d && *d->active_ref;
 }
 
 /************************************************************/
@@ -354,7 +354,7 @@
 
     /* Warning, may block waiting for rwlock_in_transaction while another
        thread runs a major GC */
-    assert(thread_descriptor->active);
+    assert(*thread_descriptor->active_ref);
     assert(in_single_thread != thread_descriptor);
 
     stm_stop_sharedlock();

From noreply at buildbot.pypy.org  Tue Oct 22 11:00:54 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Oct 2013 11:00:54 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: use new thread-locals for better
	nursery-fastpath
Message-ID: <20131022090054.8C33D1C3085@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67490:2da83847b301
Date: 2013-10-21 14:16 +0200
http://bitbucket.org/pypy/pypy/changeset/2da83847b301/

Log:	use new thread-locals for better nursery-fastpath

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -161,7 +161,7 @@
     def gen_malloc_frame(self, frame_info, frame):
         size_box = history.BoxInt()
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
-        if self.gc_ll_descr.kind == 'boehm' or self.gc_ll_descr.stm:
+        if self.gc_ll_descr.kind == 'boehm':
             op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)],
                                size_box,
                                descr=descrs.jfi_frame_depth)
@@ -171,7 +171,6 @@
             self.handle_new_array(descrs.arraydescr, op1)
         else:
             # we read size in bytes here, not the length
-            # (this path is only used in non-STM mode)
             op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)],
                                size_box,
                                descr=descrs.jfi_frame_size)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -259,10 +259,9 @@
         self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats)
         if self.cpu.gc_ll_descr.stm:
             # load nursery_current into EDI
-            self._load_stm_thread_descriptor(mc, X86_64_SCRATCH_REG)
-            mc.MOV_rm(edi.value, 
-                      (X86_64_SCRATCH_REG.value, 
-                       StmGC.TD_NURSERY_CURRENT))
+            nc = self._get_stm_tl(rstm.get_nursery_current_adr())
+            self._tl_segment_if_stm(mc)
+            mc.MOV_rj(edi.value, nc)
         else:
             nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr()
             mc.MOV(edi, heap(nursery_free_adr))   # load this in EDI
@@ -2755,25 +2754,16 @@
         # XXX if the next operation is a GUARD_NO_EXCEPTION, we should
         # somehow jump over it too in the fast path
 
-    def _load_stm_thread_descriptor(self, mc, loc):
-        assert self.cpu.gc_ll_descr.stm
-        assert isinstance(loc, RegLoc)
-        
-        td = self._get_stm_tl(rstm.get_thread_descriptor_adr())
-        self._tl_segment_if_stm(mc)
-        mc.MOV(loc, heap(td))
-        mc.MOV_rm(loc.value, (loc.value, 0))
-
+    
     def _cond_allocate_in_nursery_or_slowpath(self, mc, gcmap):
         # needed for slowpath:
         # eax = nursery_current
         # edi = nursery_current + size
-        # needed here:
-        # X86_64_SCRATCH_REG = thread_descriptor
         #
         # cmp nursery_current+size > nursery_nextlimit
-        mc.CMP_rm(edi.value, (X86_64_SCRATCH_REG.value, 
-                              StmGC.TD_NURSERY_NEXTLIMIT))
+        nnl = self._get_stm_tl(rstm.get_nursery_nextlimit_adr())
+        self._tl_segment_if_stm(mc)
+        mc.CMP_rj(edi.value, nnl)
         mc.J_il8(rx86.Conditions['NA'], 0) # patched later
         jmp_adr = mc.get_relative_pos()
         #
@@ -2781,7 +2771,7 @@
         # save the gcmap
         self.push_gcmap(mc, gcmap, mov=True)
         mc.CALL(imm(self.malloc_slowpath))
-        mc.JMP_l8(0)
+        mc.JMP_l8(0) # XXX: is JMP over 1 instr good?
         jmp2_adr = mc.get_relative_pos()
         #
         # == FASTPATH ==
@@ -2789,10 +2779,10 @@
         assert 0 < offset <= 127
         mc.overwrite(jmp_adr-1, chr(offset))
         #
-        # thread_descriptor->nursery_current = nursery_current+size
-        mc.MOV_mr((X86_64_SCRATCH_REG.value,
-                   StmGC.TD_NURSERY_CURRENT),
-                   edi.value)
+        # stm_nursery_current = stm_nursery_current+size
+        nc = self._get_stm_tl(rstm.get_nursery_current_adr())
+        self._tl_segment_if_stm(mc)
+        mc.MOV_jr(nc, edi.value)
         #
         # END
         offset = mc.get_relative_pos() - jmp2_adr
@@ -2804,10 +2794,10 @@
         assert size & (WORD-1) == 0     # must be correctly aligned
         mc = self.mc
         # load nursery_current and nursery_nextlimit
-        self._load_stm_thread_descriptor(mc, X86_64_SCRATCH_REG)
-        mc.MOV_rm(eax.value, 
-                  (X86_64_SCRATCH_REG.value,
-                   StmGC.TD_NURSERY_CURRENT))
+        nc = self._get_stm_tl(rstm.get_nursery_current_adr())
+        self._tl_segment_if_stm(mc)
+        mc.MOV_rj(eax.value, nc)
+        #
         mc.LEA_rm(edi.value, (eax.value, size))
         #
         # eax=nursery_current, edi=nursery_current+size
@@ -2816,12 +2806,14 @@
     def malloc_cond_varsize_frame_stm(self, sizeloc, gcmap):
         assert self.cpu.gc_ll_descr.stm
         mc = self.mc
-        self._load_stm_thread_descriptor(mc, X86_64_SCRATCH_REG)
         if sizeloc is eax:
             self.mc.MOV(edi, sizeloc)
             sizeloc = edi
-        self.mc.MOV_rm(eax.value, (X86_64_SCRATCH_REG.value, 
-                                   StmGC.TD_NURSERY_CURRENT))
+
+        nc = self._get_stm_tl(rstm.get_nursery_current_adr())
+        self._tl_segment_if_stm(mc)
+        mc.MOV_rj(eax.value, nc)
+        
         if sizeloc is edi:
             self.mc.ADD_rr(edi.value, eax.value)
         else:
@@ -2837,6 +2829,9 @@
         assert isinstance(arraydescr, ArrayDescr)
 
         mc = self.mc
+        nc = self._get_stm_tl(rstm.get_nursery_current_adr())
+        nnl = self._get_stm_tl(rstm.get_nursery_nextlimit_adr())
+            
         # lengthloc is the length of the array, which we must not modify!
         assert lengthloc is not eax and lengthloc is not edi
         if isinstance(lengthloc, RegLoc):
@@ -2849,10 +2844,8 @@
         mc.J_il8(rx86.Conditions['A'], 0) # patched later
         jmp_adr0 = mc.get_relative_pos()
 
-        self._load_stm_thread_descriptor(mc, X86_64_SCRATCH_REG)
-        mc.MOV_rm(eax.value, 
-                  (X86_64_SCRATCH_REG.value, 
-                   StmGC.TD_NURSERY_CURRENT))
+        self._tl_segment_if_stm(mc)
+        mc.MOV_rj(eax.value, nc)
 
         if valid_addressing_size(itemsize):
             shift = get_scale(itemsize)
@@ -2873,8 +2866,8 @@
             mc.AND_ri(edi.value, ~(WORD - 1))
         # now edi contains the total size in bytes, rounded up to a multiple
         # of WORD, plus nursery_free_adr
-        mc.CMP_rm(edi.value, (X86_64_SCRATCH_REG.value, 
-                              StmGC.TD_NURSERY_NEXTLIMIT))
+        self._tl_segment_if_stm(mc)
+        mc.CMP_rj(edi.value, nnl)
         mc.J_il8(rx86.Conditions['NA'], 0) # patched later
         jmp_adr1 = mc.get_relative_pos()
         #
@@ -2905,10 +2898,9 @@
         assert 0 < offset <= 127
         mc.overwrite(jmp_adr1-1, chr(offset))
         #
-        # set thread_descriptor->nursery_current
-        mc.MOV_mr((X86_64_SCRATCH_REG.value,
-                   StmGC.TD_NURSERY_CURRENT),
-                   edi.value)
+        # set stm_nursery_current
+        self._tl_segment_if_stm(mc)
+        mc.MOV_jr(nc, edi.value)
         #
         # write down the tid
         mc.MOV(mem(eax, 0), imm(arraydescr.tid))
diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -37,10 +37,6 @@
     malloc_zero_filled = True
     #gcflag_extra = GCFLAG_EXTRA
 
-    # SYNC with et.h
-    TD_NURSERY_CURRENT = 80
-    TD_NURSERY_NEXTLIMIT = 88
-    
     GCHDR = lltype.Struct(
         'GCPTR',
         ('h_tid', lltype.Unsigned),
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -5,8 +5,18 @@
 from rpython.rlib.jit import dont_look_inside
 
 @dont_look_inside
-def get_thread_descriptor_adr():
-    addr = llop.stm_get_adr_of_thread_descriptor(llmemory.Address)
+def get_nursery_current_adr():
+    addr = llop.stm_get_adr_of_nursery_current(llmemory.Address)
+    return rffi.cast(lltype.Signed, addr)
+
+ at dont_look_inside
+def get_nursery_nextlimit_adr():
+    addr = llop.stm_get_adr_of_nursery_nextlimit(llmemory.Address)
+    return rffi.cast(lltype.Signed, addr)
+
+ at dont_look_inside
+def get_active_adr():
+    addr = llop.stm_get_adr_of_active(llmemory.Address)
     return rffi.cast(lltype.Signed, addr)
 
 @dont_look_inside
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -951,7 +951,9 @@
     op_stm_barrier = _stm_not_implemented
     op_stm_push_root = _stm_not_implemented
     op_stm_pop_root_into = _stm_not_implemented
-    op_stm_get_adr_of_thread_descriptor = _stm_not_implemented
+    op_stm_get_adr_of_nursery_current = _stm_not_implemented
+    op_stm_get_adr_of_nursery_nextlimit = _stm_not_implemented
+    op_stm_get_adr_of_active = _stm_not_implemented
     op_stm_get_adr_of_read_barrier_cache = _stm_not_implemented
     op_stm_get_adr_of_private_rev_num = _stm_not_implemented
     op_stm_enter_callback_call = _stm_not_implemented
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -448,7 +448,9 @@
 
     'stm_get_adr_of_private_rev_num':LLOp(),
     'stm_get_adr_of_read_barrier_cache':LLOp(),
-    'stm_get_adr_of_thread_descriptor': LLOp(),
+    'stm_get_adr_of_nursery_current': LLOp(),
+    'stm_get_adr_of_nursery_nextlimit': LLOp(),
+    'stm_get_adr_of_active': LLOp(),
 
     'stm_ignored_start':      LLOp(canrun=True),
     'stm_ignored_stop':       LLOp(canrun=True),
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -590,7 +590,9 @@
     OP_STM_PTR_EQ                       = _OP_STM
     OP_STM_PUSH_ROOT                    = _OP_STM
     OP_STM_POP_ROOT_INTO                = _OP_STM
-    OP_STM_GET_ADR_OF_THREAD_DESCRIPTOR = _OP_STM
+    OP_STM_GET_ADR_OF_NURSERY_CURRENT   = _OP_STM
+    OP_STM_GET_ADR_OF_NURSERY_NEXTLIMIT = _OP_STM
+    OP_STM_GET_ADR_OF_ACTIVE            = _OP_STM
     OP_STM_GET_ROOT_STACK_TOP           = _OP_STM
     OP_STM_GET_ADR_OF_PRIVATE_REV_NUM   = _OP_STM
     OP_STM_GET_ADR_OF_READ_BARRIER_CACHE= _OP_STM
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -114,9 +114,19 @@
     return '%s = (%s)stm_pop_root();' % (
         arg0, cdecl(funcgen.lltypename(op.args[0]), ''))
 
-def stm_get_adr_of_thread_descriptor(funcgen, op):
+def stm_get_adr_of_nursery_current(funcgen, op):
     result = funcgen.expr(op.result)
-    return '%s = (%s)&thread_descriptor;' % (
+    return '%s = (%s)&stm_nursery_current;' % (
+        result, cdecl(funcgen.lltypename(op.result), ''))
+
+def stm_get_adr_of_nursery_nextlimit(funcgen, op):
+    result = funcgen.expr(op.result)
+    return '%s = (%s)&stm_nursery_nextlimit;' % (
+        result, cdecl(funcgen.lltypename(op.result), ''))
+
+def stm_get_adr_of_active(funcgen, op):
+    result = funcgen.expr(op.result)
+    return '%s = (%s)&stm_active;' % (
         result, cdecl(funcgen.lltypename(op.result), ''))
     
 def stm_get_root_stack_top(funcgen, op):

From noreply at buildbot.pypy.org  Tue Oct 22 11:00:56 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Oct 2013 11:00:56 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: make transaction breaks after calls
 simply check for stm_active==2 in order to
Message-ID: <20131022090056.20CD51C3085@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67491:d5878ec15352
Date: 2013-10-21 14:38 +0200
http://bitbucket.org/pypy/pypy/changeset/d5878ec15352/

Log:	make transaction breaks after calls simply check for stm_active==2
	in order to break the transaction if it is inevitable

diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -90,7 +90,8 @@
                     and insert_transaction_break):
                     # insert transaction_break after GUARD after calls
                     self.newops.append(
-                        ResOperation(rop.STM_TRANSACTION_BREAK, [], None))
+                        ResOperation(rop.STM_TRANSACTION_BREAK,
+                                     [ConstInt(0)], None))
                     insert_transaction_break = False
                     self.emitting_an_operation_that_can_collect()
                     self.next_op_may_be_in_new_transaction()
@@ -169,7 +170,8 @@
             # ----------  jumps  ----------
             if op.getopnum() == rop.JUMP:
                 self.newops.append(
-                    ResOperation(rop.STM_TRANSACTION_BREAK, [], None))
+                    ResOperation(rop.STM_TRANSACTION_BREAK,
+                                 [ConstInt(1)], None))
                 # self.emitting_an_operation_that_can_collect()
                 self.newops.append(op)
                 continue
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -82,7 +82,7 @@
                 []
                 %s
                 call(123, descr=cd)
-                stm_transaction_break()
+                stm_transaction_break(1)
                 jump()
             """ % ("$INEV" if inev else "",), cd=calldescr)
     
@@ -95,7 +95,7 @@
             [p1, p2]
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, p2, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -111,7 +111,7 @@
             p3 = same_as(ConstPtr(t))
             cond_call_stm_b(p3, descr=P2Wdescr)
             setfield_gc(p3, p2, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
             """, t=NULL)
 
@@ -136,7 +136,7 @@
             setfield_gc(p2, p0, descr=tzdescr)
             cond_call_stm_b(p1, descr=P2Rdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
             """, t=NULL)
 
@@ -159,7 +159,7 @@
             setfield_gc(p2, p0, descr=tzdescr)
             cond_call_stm_b(p1, descr=P2Rdescr)
             p5 = getfield_gc(p1, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -183,7 +183,7 @@
             setfield_gc(p2, p0, descr=tzdescr)
             cond_call_stm_b(p1, descr=P2Rdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -209,7 +209,7 @@
             setfield_gc(p2, p0, descr=tzdescr)
             cond_call_stm_b(p1, descr=P2Rdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """
         for op in ops:
@@ -232,7 +232,7 @@
             stm_set_revision_gc(p2, descr=revdescr)
             cond_call_stm_b(p3, descr=P2Wdescr)
             setfield_gc(p3, p1, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p2)
         """)
 
@@ -251,7 +251,7 @@
             setfield_gc(p3, %(tdescr.tid)d, descr=tiddescr)
             stm_set_revision_gc(p3, descr=revdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p2)
         """)
             
@@ -267,7 +267,7 @@
             setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr)
             stm_set_revision_gc(p2, descr=revdescr)
             setfield_gc(p2, p1, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p2)
         """)
 
@@ -283,7 +283,7 @@
             setfield_gc(p1, p2, descr=tzdescr)
             cond_call_stm_b(p3, descr=P2Wdescr)
             setfield_gc(p3, p4, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -298,7 +298,7 @@
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, p2, descr=tzdescr)
             setfield_gc(p1, i3, descr=tydescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -316,7 +316,7 @@
             label(p1, i3)
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, i3, descr=tydescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p1)
         """)
 
@@ -327,7 +327,7 @@
             jump()
         """, """
             [i1, i2]
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -360,7 +360,7 @@
             [p1]
             cond_call_stm_b(p1, descr=P2Rdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p2)
         """)
 
@@ -376,7 +376,7 @@
             p3 = same_as(ConstPtr(t))
             cond_call_stm_b(p3, descr=P2Rdescr)
             p2 = getfield_gc(p3, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p2)
         """, t=NULL)
         # XXX could do better: G2Rdescr
@@ -390,7 +390,7 @@
             [p1, i2]
             cond_call_stm_b(p1, descr=P2Rdescr)
             i3 = getarrayitem_gc(p1, i2, descr=adescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i3)
         """)
 
@@ -403,7 +403,7 @@
             [p1, i2]
             cond_call_stm_b(p1, descr=P2Rdescr)
             i3 = getinteriorfield_gc(p1, i2, descr=adescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i3)
         """)
 
@@ -418,7 +418,7 @@
             cond_call_stm_b(p1, descr=P2Rdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
             i2 = getfield_gc(p1, descr=tydescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p2, i2)
         """)
 
@@ -434,7 +434,7 @@
             p2 = getfield_gc(p1, descr=tzdescr)
             cond_call_stm_b(p2, descr=P2Rdescr)
             i2 = getfield_gc(p2, descr=tydescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p2, i2)
         """)
 
@@ -454,7 +454,7 @@
             i2 = int_add(i1, 1)
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, i2, descr=tydescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p1)
         """)
 
@@ -469,7 +469,7 @@
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, 123, descr=tydescr)
             p2 = getfield_gc(p1, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p2)
         """)
 
@@ -485,7 +485,7 @@
             setfield_gc(p1, %(tdescr.tid)d, descr=tiddescr)
             stm_set_revision_gc(p1, descr=revdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p2)
         """)
 
@@ -511,7 +511,7 @@
             call(p2, descr=calldescr1)
             cond_call_stm_b(p1, descr=P2Wdescr)
             setfield_gc(p1, 5, descr=tydescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(p2)
         """, calldescr1=calldescr1)
 
@@ -528,7 +528,7 @@
             i3 = getfield_raw(i1, descr=tydescr)
             keepalive(i3)
             i4 = getfield_raw(i2, descr=tydescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i3, i4)
         """)
 
@@ -544,7 +544,7 @@
         """, """
             [i1]
             i2 = getfield_raw(i1, descr=fdescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i2)
         """, fdescr=fdescr)
 
@@ -562,7 +562,7 @@
             label(i1, i2, i3)
             $INEV
             i4 = getfield_raw(i2, descr=tydescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i3, i4)
         """)
 
@@ -577,7 +577,7 @@
             $INEV
             i3 = getarrayitem_raw(i1, 5, descr=adescr)
             i4 = getarrayitem_raw(i2, i3, descr=adescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i3, i4)
         """)
 
@@ -593,7 +593,7 @@
             setarrayitem_gc(p1, i1, p2, descr=adescr)
             cond_call_stm_b(p3, descr=P2Wdescr)
             setarrayitem_gc(p3, i3, p4, descr=adescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -610,7 +610,7 @@
             setarrayitem_gc(p1, i2, p2, descr=adescr)
             i4 = read_timestamp()
             setarrayitem_gc(p1, i3, p3, descr=adescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -627,7 +627,7 @@
             setinteriorfield_gc(p1, i2, p2, descr=adescr)
             i4 = read_timestamp()
             setinteriorfield_gc(p1, i3, p3, descr=adescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -642,7 +642,7 @@
             cond_call_stm_b(p1, descr=P2Wdescr)
             strsetitem(p1, i2, i3)
             unicodesetitem(p1, i2, i3)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
         py.test.skip("XXX not really right: should instead be an assert "
@@ -664,10 +664,10 @@
             setfield_gc(p7, 10, descr=tydescr)
             call_release_gil(123, descr=calldescr2)
             guard_not_forced() []
-            stm_transaction_break()
+            stm_transaction_break(0)
             cond_call_stm_b(p7, descr=P2Wdescr)
             setfield_gc(p7, 20, descr=tydescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i2, p7)
         """, calldescr2=calldescr2)
         
@@ -695,7 +695,7 @@
                 %s
                 cond_call_stm_b(p7, descr=P2Wdescr)
                 setfield_gc(p7, 20, descr=tydescr)
-                stm_transaction_break()
+                stm_transaction_break(1)
                 jump(i2, p7)
             """ % op, calldescr2=calldescr2)
 
@@ -709,7 +709,7 @@
             cond_call_stm_b(p2, descr=P2Wdescr)
             cond_call_stm_b(p1, descr=P2Rdescr)
             copystrcontent(p1, p2, i1, i2, i3)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -731,7 +731,7 @@
                 setfield_gc(p1, 10, descr=tydescr)
                 %s
                 setfield_gc(p1, 20, descr=tydescr)
-                stm_transaction_break()
+                stm_transaction_break(1)
                 jump(p1)
             """ % op)
 
@@ -748,7 +748,7 @@
                 ("call_loopinvariant(123, descr=calldescr2)", False),
                 ]:
             guard = "guard_not_forced() []" if guarded else ""
-            tr_break = "stm_transaction_break()" if guarded else ""
+            tr_break = "stm_transaction_break(0)" if guarded else ""
             self.check_rewrite("""
                 [p1]
                 setfield_gc(p1, 10, descr=tydescr)
@@ -765,7 +765,7 @@
                 %s
                 cond_call_stm_b(p1, descr=P2Wdescr)
                 setfield_gc(p1, 20, descr=tydescr)
-                stm_transaction_break()
+                stm_transaction_break(1)
                 jump(p1)
             """ % (op, guard, tr_break), calldescr2=calldescr2)
 
@@ -787,7 +787,7 @@
         setarrayitem_gc(p1, 1, f0, descr=floatframedescr)
         i3 = call_assembler(p1, descr=casmdescr)
         guard_not_forced() []
-        stm_transaction_break()
+        stm_transaction_break(0)
         """)
 
     def test_ptr_eq_null(self):
@@ -798,7 +798,7 @@
         """, """
             [p1, p2]
             i1 = ptr_eq(p1, NULL)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i1)
         """)
 
@@ -810,7 +810,7 @@
         """, """
             [p1, p2]
             i1 = ptr_eq(p1, p2)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i1)
         """)
 
@@ -822,7 +822,7 @@
         """, """
             [p1, p2]
             i1 = instance_ptr_eq(p1, p2)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i1)
         """)
 
@@ -834,7 +834,7 @@
         """, """
             [p1, p2]
             i1 = ptr_ne(p1, p2)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i1)
         """)
 
@@ -846,7 +846,7 @@
         """, """
             [p1, p2]
             i1 = instance_ptr_ne(p1, p2)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i1)
         """)
 
@@ -980,7 +980,7 @@
             [i0]
             p0 = call_malloc_nursery_varsize(0, 1, i0, descr=bdescr)
             setfield_gc(p0, i0, descr=blendescr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i0)
         """)
 
@@ -993,7 +993,7 @@
         [i0]
         p0 = call_malloc_nursery_varsize(1, 1, i0, descr=strdescr)
         setfield_gc(p0, i0, descr=strlendescr)
-        stm_transaction_break()
+        stm_transaction_break(1)
         jump(i0)
         """)
 
@@ -1017,7 +1017,7 @@
                                 %(nonstd_descr.lendescr.offset)d,     \
                                 6464, i0,                             \
                                 descr=malloc_array_nonstandard_descr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump(i0)
         """, nonstd_descr=nonstd_descr)
 
@@ -1032,7 +1032,7 @@
             p0 = call_malloc_gc(ConstClass(malloc_array), 1,  \
                                 %(bdescr.tid)d, 103,          \
                                 descr=malloc_array_descr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -1072,7 +1072,7 @@
             p0 = call_malloc_gc(ConstClass(malloc_array), 1, \
                                 %(bdescr.tid)d, 20000000,    \
                                 descr=malloc_array_descr)
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """)
 
@@ -1175,7 +1175,7 @@
             
             call_may_force(123, descr=calldescr2)
             guard_not_forced() []
-            stm_transaction_break()
+            stm_transaction_break(0)
             
             p2 = call_malloc_nursery(    \
                               %(bdescr.basesize + 8)d)
@@ -1211,7 +1211,7 @@
             []
             call_may_force(123, descr=calldescr2)
             guard_not_forced() []
-            stm_transaction_break()
+            stm_transaction_break(0)
             
             label()
             
@@ -1220,7 +1220,7 @@
 
             i0 = int_add(1, 2)
             
-            stm_transaction_break()
+            stm_transaction_break(1)
             jump()
         """, calldescr2=calldescr2)
 
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -3065,11 +3065,23 @@
         assert self.cpu.gc_ll_descr.stm
         if not we_are_translated():
             return     # tests only
-        # "if stm_should_break_transaction()"
+
+        # argument is check_type, 0 being a check for inevitable,
+        # 1 being a check with stm_should_break_transaction()
+        assert isinstance(arglocs[0], ImmedLoc)
+        check_type = arglocs[0].getint()
+        
         mc = self.mc
-        fn = stmtlocal.stm_should_break_transaction_fn
-        mc.CALL(imm(self.cpu.cast_ptr_to_int(fn)))
-        mc.TEST8(eax.lowest8bits(), eax.lowest8bits())
+        if check_type == 0:
+            # if stm_active == 2
+            nc = self._get_stm_tl(rstm.get_active_adr())
+            self._tl_segment_if_stm(mc)
+            mc.CMP_ji(nc, 2)
+        elif check_type == 1:
+            # "if stm_should_break_transaction()"
+            fn = stmtlocal.stm_should_break_transaction_fn
+            mc.CALL(imm(self.cpu.cast_ptr_to_int(fn)))
+            mc.TEST8(eax.lowest8bits(), eax.lowest8bits())
         mc.J_il(rx86.Conditions['Z'], 0xfffff)    # patched later
         jz_location = mc.get_relative_pos()
         #
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1286,9 +1286,11 @@
     def consider_stm_transaction_break(self, op):
         # XXX use the extra 3 words in the stm resume buffer to save
         # up to 3 registers, too.  For now we just flush them all.
+        check_type = op.getarg(0)
+        assert isinstance(check_type, ConstInt)
         self.xrm.before_call(save_all_regs=1)
         self.rm.before_call(save_all_regs=1)
-        self.perform(op, [], None)
+        self.perform(op, [self.rm.convert_to_imm(check_type)], None)
 
     def consider_jump(self, op):
         assembler = self.assembler
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -510,7 +510,7 @@
     'QUASIIMMUT_FIELD/1d',    # [objptr], descr=SlowMutateDescr
     'RECORD_KNOWN_CLASS/2',   # [objptr, clsptr]
     'KEEPALIVE/1',
-    'STM_TRANSACTION_BREAK/0',
+    'STM_TRANSACTION_BREAK/1',
     'STM_SET_REVISION_GC/1d', # not really GC, writes raw to the header
 
     '_CANRAISE_FIRST', # ----- start of can_raise operations -----

From noreply at buildbot.pypy.org  Tue Oct 22 11:00:57 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Oct 2013 11:00:57 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: move the call to
 stm_transaction_break() into separately piece (not inlined in
Message-ID: <20131022090057.7198B1C3085@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67492:a58510da97be
Date: 2013-10-21 20:19 +0200
http://bitbucket.org/pypy/pypy/changeset/a58510da97be/

Log:	move the call to stm_transaction_break() into separately piece (not
	inlined in trace anymore)

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -95,6 +95,7 @@
         if gc_ll_descr.stm:
             self._build_ptr_eq_slowpath()
             self._build_stm_longjmp_callback()
+            self.stm_transaction_break_path = self._build_stm_transaction_break_path()
         else:
             self.ptr_eq_slowpath = None
         # only one of those
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -161,7 +161,7 @@
     def gen_malloc_frame(self, frame_info, frame):
         size_box = history.BoxInt()
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
-        if self.gc_ll_descr.kind == 'boehm':
+        if self.gc_ll_descr.kind == 'boehm' or self.gc_ll_descr.stm:
             op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)],
                                size_box,
                                descr=descrs.jfi_frame_depth)
@@ -171,6 +171,7 @@
             self.handle_new_array(descrs.arraydescr, op1)
         else:
             # we read size in bytes here, not the length
+            # jfi_frame_size not set in STM!
             op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)],
                                size_box,
                                descr=descrs.jfi_frame_size)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -188,6 +188,50 @@
         mc.RET()
         return mc.materialize(self.cpu.asmmemmgr, [])
 
+    def _build_stm_transaction_break_path(self):
+        """ While arriving on slowpath, we have a gcpattern on stack 0.
+        This function must preserve all registers
+        """
+        mc = codebuf.MachineCodeBlockWrapper()
+        # store the gc pattern
+        ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap')
+        mc.MOV_rs(ecx.value, WORD)
+        mc.MOV_br(ofs, ecx.value)
+        #
+        # align on 16b boundary (there is a retaddr on the stack)
+        mc.SUB_ri(esp.value, 16 - WORD)
+        #
+        # call stm_transaction_break() with the address of the
+        # STM_RESUME_BUF and the custom longjmp function
+        # (rsp + FRAME_FIXED_SIZE + RET_ADDR + ALIGNMENT)
+        mc.LEA_rs(edi.value, (FRAME_FIXED_SIZE+2) * WORD)
+        mc.MOV(esi, imm(self.stm_longjmp_callback_addr))
+        fn = stmtlocal.stm_transaction_break_fn
+        mc.CALL(imm(self.cpu.cast_ptr_to_int(fn)))
+        #
+        mc.ADD_ri(esp.value, 16 - WORD)
+        #
+        self._reload_frame_if_necessary(mc, align_stack=True)
+        # clear the gc pattern
+        mc.MOV_bi(ofs, 0)
+        #
+        # Fill the stm resume buffer.  Don't do it before the call!
+        # The previous transaction may still be aborted during the call
+        # above, so we need the old content of the buffer!
+        # The buffer contains the address of the resume point which
+        # is the RET_ADDR of this called piece of code. This will be
+        # put at offset 0 of the buffer, at offset WORD, there is a
+        # copy of the current shadowstack pointer.
+        mc.POP_r(eax.value) # get ret addr
+        self._load_shadowstack_top_in_ebx(mc, self.cpu.gc_ll_descr.gcrootmap)
+        mc.MOV_sr((FRAME_FIXED_SIZE + 1) * WORD, ebx.value)
+        mc.MOV_sr((FRAME_FIXED_SIZE + 0) * WORD, eax.value)
+        mc.JMP_r(eax.value)
+        #
+        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        return rawstart
+
+    
     def _build_malloc_slowpath(self, kind):
         """ While arriving on slowpath, we have a gcpattern on stack 0.
         The arguments are passed in eax and edi, as follows:
@@ -3061,54 +3105,46 @@
         mc.MOV(dest_addr, X86_64_SCRATCH_REG)
 
         
-    def genop_stm_transaction_break(self, op, arglocs, result_loc):
+    def stm_transaction_break(self, check_type, gcmap):
         assert self.cpu.gc_ll_descr.stm
         if not we_are_translated():
             return     # tests only
 
-        # argument is check_type, 0 being a check for inevitable,
-        # 1 being a check with stm_should_break_transaction()
-        assert isinstance(arglocs[0], ImmedLoc)
-        check_type = arglocs[0].getint()
-        
+        # check_type: 0 do a check for inevitable before
+        # doing a check of stm_should_break_transaction().
+        # else, just do stm_should_break_transaction()
         mc = self.mc
         if check_type == 0:
-            # if stm_active == 2
+            # only check stm_should_break_transaction()
+            # if we are inevitable:
             nc = self._get_stm_tl(rstm.get_active_adr())
             self._tl_segment_if_stm(mc)
-            mc.CMP_ji(nc, 2)
-        elif check_type == 1:
-            # "if stm_should_break_transaction()"
-            fn = stmtlocal.stm_should_break_transaction_fn
-            mc.CALL(imm(self.cpu.cast_ptr_to_int(fn)))
-            mc.TEST8(eax.lowest8bits(), eax.lowest8bits())
+            mc.CMP_ji(nc, 1)
+            mc.J_il(rx86.Conditions['Z'], 0xfffff)    # patched later
+            jz_location = mc.get_relative_pos()
+        else:
+            jz_location = 0
+        
+        # if stm_should_break_transaction()
+        fn = stmtlocal.stm_should_break_transaction_fn
+        mc.CALL(imm(self.cpu.cast_ptr_to_int(fn)))
+        mc.TEST8(eax.lowest8bits(), eax.lowest8bits())
         mc.J_il(rx86.Conditions['Z'], 0xfffff)    # patched later
-        jz_location = mc.get_relative_pos()
+        jz_location2 = mc.get_relative_pos()
         #
         # call stm_transaction_break() with the address of the
         # STM_RESUME_BUF and the custom longjmp function
-        mc.LEA_rs(edi.value, FRAME_FIXED_SIZE * WORD)
-        fn = stmtlocal.stm_transaction_break_fn
-        self.simple_call(imm(self.cpu.cast_ptr_to_int(fn)),
-                         [edi, imm(self.stm_longjmp_callback_addr)],
-                         None)
-        #
-        # Fill the stm resume buffer.  Don't do it before the call!
-        # The previous transaction may still be aborted during the call
-        # above, so we need the old content of the buffer!
-        # The buffer contains the address of the resume point in this
-        # piece of code (at "HERE") at offset 0, and at offset WORD it
-        # contains a copy of the current shadowstack pointer.
-        self._load_shadowstack_top_in_ebx(mc, self.cpu.gc_ll_descr.gcrootmap)
-        mc.MOV_sr((FRAME_FIXED_SIZE + 1) * WORD, ebx.value)
-        mc.CALL_l(0)
-        # "HERE"
-        mc.POP_r(eax.value)
-        mc.MOV_sr((FRAME_FIXED_SIZE + 0) * WORD, eax.value)
+        self.push_gcmap(mc, gcmap, mov=True)
+        fn = self.stm_transaction_break_path
+        mc.CALL(imm(fn))
         #
         # patch the JZ above
-        offset = mc.get_relative_pos() - jz_location
-        mc.overwrite32(jz_location-4, offset)
+        if jz_location:
+            offset = mc.get_relative_pos() - jz_location
+            mc.overwrite32(jz_location-4, offset)
+        offset = mc.get_relative_pos() - jz_location2
+        mc.overwrite32(jz_location2-4, offset)
+
 
 
 genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1284,13 +1284,18 @@
         self.perform_discard(op, [base_loc, ofs_loc, size_loc])
         
     def consider_stm_transaction_break(self, op):
+        check_type_box = op.getarg(0)
+        assert isinstance(check_type_box, ConstInt)
+        check_type = check_type_box.getint()
+        #
         # XXX use the extra 3 words in the stm resume buffer to save
         # up to 3 registers, too.  For now we just flush them all.
-        check_type = op.getarg(0)
-        assert isinstance(check_type, ConstInt)
         self.xrm.before_call(save_all_regs=1)
         self.rm.before_call(save_all_regs=1)
-        self.perform(op, [self.rm.convert_to_imm(check_type)], None)
+        gcmap = self.get_gcmap() # allocate the gcmap *before*
+        #
+        self.assembler.stm_transaction_break(check_type, gcmap)
+        
 
     def consider_jump(self, op):
         assembler = self.assembler

From noreply at buildbot.pypy.org  Tue Oct 22 11:00:58 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Oct 2013 11:00:58 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: fix now unnecessary PUSH
Message-ID: <20131022090058.985231C3085@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67493:75d0f6b0579d
Date: 2013-10-22 10:59 +0200
http://bitbucket.org/pypy/pypy/changeset/75d0f6b0579d/

Log:	fix now unnecessary PUSH

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -204,14 +204,14 @@
         # call stm_transaction_break() with the address of the
         # STM_RESUME_BUF and the custom longjmp function
         # (rsp + FRAME_FIXED_SIZE + RET_ADDR + ALIGNMENT)
-        mc.LEA_rs(edi.value, (FRAME_FIXED_SIZE+2) * WORD)
+        mc.LEA_rs(edi.value, FRAME_FIXED_SIZE * WORD + WORD + (16-WORD))
         mc.MOV(esi, imm(self.stm_longjmp_callback_addr))
         fn = stmtlocal.stm_transaction_break_fn
         mc.CALL(imm(self.cpu.cast_ptr_to_int(fn)))
         #
+        self._reload_frame_if_necessary(mc)
+        #
         mc.ADD_ri(esp.value, 16 - WORD)
-        #
-        self._reload_frame_if_necessary(mc, align_stack=True)
         # clear the gc pattern
         mc.MOV_bi(ofs, 0)
         #
@@ -632,7 +632,6 @@
         # jump to the place saved in stm_resume_buffer[0]
         # (to "HERE" in genop_stm_transaction_break())
         mc.MOV_rs(eax.value, (FRAME_FIXED_SIZE + 0) * WORD)
-        mc.PUSH_r(eax.value)
         mc.JMP_r(eax.value)
         self.stm_longjmp_callback_addr = mc.materialize(self.cpu.asmmemmgr, [])
 

From noreply at buildbot.pypy.org  Tue Oct 22 11:33:21 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 22 Oct 2013 11:33:21 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: (fijal,
 arigo) The goal is to support overallocated arrays more directly in
Message-ID: <20131022093321.E65331C0149@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: array-overallocation-in-nursery
Changeset: r67494:2cd6bc0f2b06
Date: 2013-10-22 11:30 +0200
http://bitbucket.org/pypy/pypy/changeset/2cd6bc0f2b06/

Log:	(fijal, arigo) The goal is to support overallocated arrays more
	directly in the GC. The lltype simulation part

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1,6 +1,6 @@
 from rpython.rlib.rarithmetic import (r_int, r_uint, intmask, r_singlefloat,
-                                   r_ulonglong, r_longlong, r_longfloat, r_longlonglong,
-                                   base_int, normalizedinttype, longlongmask, longlonglongmask)
+     r_ulonglong, r_longlong, r_longfloat, r_longlonglong, base_int,
+     normalizedinttype, longlongmask, longlonglongmask)
 from rpython.rlib.objectmodel import Symbolic
 from rpython.tool.identity_dict import identity_dict
 from rpython.tool import leakfinder
@@ -193,6 +193,9 @@
     def _inline_is_varsize(self, last):
         raise TypeError, "%r cannot be inlined in structure" % self
 
+    def _is_overallocated_array(self):
+        return False
+
     def _install_extras(self, adtmeths={}, hints={}):
         self._adtmeths = frozendict(adtmeths)
         self._hints = frozendict(hints)
@@ -435,6 +438,11 @@
         self.OF._inline_is_varsize(False)
 
         self._install_extras(**kwds)
+        if self._is_overallocated_array():
+            assert self._gckind == 'gc', "no support for raw overallocated arrays"
+
+    def _is_overallocated_array(self):
+        return self._hints.get('overallocated', False)
 
     def _inline_is_varsize(self, last):
         if not last:
@@ -481,7 +489,6 @@
     def _inline_is_varsize(self, last):
         raise TypeError("cannot inline a GC array inside a structure")
 
-
 class FixedSizeArray(Struct):
     # behaves more or less like a Struct with fields item0, item1, ...
     # but also supports __getitem__(), __setitem__(), __len__().
@@ -1160,6 +1167,11 @@
             if field_name in self._T._flds:
                 o = self._obj._getattr(field_name)
                 return self._expose(field_name, o)
+        if self._T._is_overallocated_array():
+            if field_name == 'allocated_length':
+                return self._obj.getlength()
+            elif field_name == 'used_length':
+                return self._obj.used_len
         try:
             return self._lookup_adtmeth(field_name)
         except AttributeError:
@@ -1178,6 +1190,16 @@
                                     "expects %r\n"
                                     "    got %r" % (self._T, field_name, T1, T2))
                 return
+        if self._T._is_overallocated_array():
+            if field_name == 'used_length':
+                if val > self._obj.getlength():
+                    raise ValueError("overallocated array size is %s, trying "
+                                     "to set used size to %s" %
+                                     (self._obj.getlength(), val))
+                for i in range(val, self._obj.used_len):
+                    self._obj.items[i] = self._T.OF._allocate('malloc')
+                self._obj.used_len = val
+                return
         raise AttributeError("%r instance has no field %r" % (self._T,
                                                               field_name))
 
@@ -1218,7 +1240,8 @@
 
     def __len__(self):
         if isinstance(self._T, (Array, FixedSizeArray)):
-            if self._T._hints.get('nolength', False):
+            if (self._T._hints.get('nolength', False) or
+                self._T._hints.get('overallocated', False)):
                 raise TypeError("%r instance has no length attribute" %
                                     (self._T,))
             return self._obj.getlength()
@@ -1652,14 +1675,14 @@
 class _array(_parentable):
     _kind = "array"
 
-    __slots__ = ('items',)
-
     def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None):
         if not is_valid_int(n):
             raise TypeError, "array length must be an int"
         if n < 0:
             raise ValueError, "negative array length"
         _parentable.__init__(self, TYPE)
+        if TYPE._is_overallocated_array():
+            self.used_len = 0
         myrange = self._check_range(n)
         self.items = [TYPE.OF._allocate(initialization=initialization,
                                         parent=self, parentindex=j)
@@ -1709,7 +1732,10 @@
         del self.items[newlength:]
 
     def getbounds(self):
-        stop = len(self.items)
+        if self._TYPE._is_overallocated_array():
+            stop = self.used_len
+        else:
+            stop = len(self.items)
         return 0, stop
 
     def getitem(self, index, uninitialized_ok=False):
@@ -1722,7 +1748,6 @@
         assert typeOf(value) == self._TYPE.OF
         self.items[index] = value
 
-assert not '__dict__' in dir(_array)
 assert not '__dict__' in dir(_struct)
 
 
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -235,7 +235,7 @@
     del p1
     import gc
     gc.collect()
-    py.test.raises(RuntimeError, "p1_5.v")        
+    py.test.raises(RuntimeError, "p1_5.v")
 
 def test_examples():
     A1 = GcArray(('v', Signed))
@@ -388,11 +388,11 @@
         s.x = 1
     def type_info_S(p):
         return getRuntimeTypeInfo(S)
-    qp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)), 
-                     "type_info_S", 
+    qp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)),
+                     "type_info_S",
                      _callable=type_info_S)
-    dp = functionptr(FuncType([Ptr(S)], Void), 
-                     "destructor_funcptr", 
+    dp = functionptr(FuncType([Ptr(S)], Void),
+                     "destructor_funcptr",
                      _callable=f)
     pinf0 = attachRuntimeTypeInfo(S, qp, destrptr=dp)
     assert pinf0._obj.about == S
@@ -422,8 +422,8 @@
             return getRuntimeTypeInfo(S)
         else:
             return getRuntimeTypeInfo(S1)
-    fp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)), 
-                     "dynamic_type_info_S", 
+    fp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)),
+                     "dynamic_type_info_S",
                      _callable=dynamic_type_info_S)
     attachRuntimeTypeInfo(S, fp)
     assert s.x == 0
@@ -434,7 +434,7 @@
     py.test.raises(RuntimeError, "runtime_type_info(s1.sub)")
     s1.sub.x = 1
     assert runtime_type_info(s1.sub) == getRuntimeTypeInfo(S1)
-    
+
 def test_flavor_malloc():
     def isweak(p, T):
         return p._weak and typeOf(p).TO == T
@@ -450,7 +450,7 @@
     p = malloc(T, flavor="gc")
     assert typeOf(p).TO == T
     assert not isweak(p, T)
-    
+
 def test_opaque():
     O = OpaqueType('O')
     p1 = opaqueptr(O, 'p1', hello="world")
@@ -520,8 +520,8 @@
 def test_adtmeths():
     def h_newstruct():
         return malloc(S)
-    
-    S = GcStruct('s', ('x', Signed), 
+
+    S = GcStruct('s', ('x', Signed),
                  adtmeths={"h_newstruct": h_newstruct})
 
     s = S.h_newstruct()
@@ -553,15 +553,15 @@
     def h_newstruct(S):
         return malloc(S)
     h_newstruct = typeMethod(h_newstruct)
-    
-    S = GcStruct('s', ('x', Signed), 
+
+    S = GcStruct('s', ('x', Signed),
                  adtmeths={"h_newstruct": h_newstruct})
 
     s = S.h_newstruct()
 
     assert typeOf(s) == Ptr(S)
 
-    Sprime = GcStruct('s', ('x', Signed), 
+    Sprime = GcStruct('s', ('x', Signed),
                       adtmeths={"h_newstruct": h_newstruct})
 
     assert S == Sprime
@@ -592,7 +592,7 @@
     PA = Ptr(A)
     a = malloc(A, 2)
     assert cast_pointer(PA, a) == a
-        
+
 def test_array_with_no_length():
     A = GcArray(Signed, hints={'nolength': True})
     a = malloc(A, 10)
@@ -604,7 +604,7 @@
     s = malloc(GcS)
     s.x = 1
     assert list(dissect_ll_instance(s)) == [(Ptr(GcS), s), (GcS, s._obj), (Signed, 1)]
-    
+
     A = GcArray(('x', Signed))
     a = malloc(A, 10)
     for i in range(10):
@@ -808,6 +808,21 @@
     assert F.RESULT == Signed
     assert F.ARGS == (Signed,)
 
+def test_overallocated_array():
+    A = GcArray(lltype.Signed, hints={'overallocated': True})
+    a = lltype.malloc(A, 10)
+    py.test.raises(IndexError, "a[1]")
+    py.test.raises(TypeError, len, a)
+    assert a.allocated_length == 10
+    assert a.used_length == 0
+    a.used_length = 5
+    py.test.raises(ValueError, "a.used_length = 13")
+    a[3] = 42
+    assert a[3] == 42
+    a.used_length = 1
+    py.test.raises(IndexError, "a[3]")
+    py.test.raises(IndexError, "a[3] = 43")
+
 class TestTrackAllocation:
     def test_automatic_tracking(self):
         # calls to start_tracking_allocations/stop_tracking_allocations

From noreply at buildbot.pypy.org  Tue Oct 22 11:33:23 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 22 Oct 2013 11:33:23 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: improve the test
Message-ID: <20131022093323.675771C0149@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: array-overallocation-in-nursery
Changeset: r67495:877b9bad519b
Date: 2013-10-22 11:31 +0200
http://bitbucket.org/pypy/pypy/changeset/877b9bad519b/

Log:	improve the test

diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -822,6 +822,8 @@
     a.used_length = 1
     py.test.raises(IndexError, "a[3]")
     py.test.raises(IndexError, "a[3] = 43")
+    a.used_length = 5
+    py.test.raises(UninitializedMemoryAccess, "a[3]")
 
 class TestTrackAllocation:
     def test_automatic_tracking(self):

From noreply at buildbot.pypy.org  Tue Oct 22 11:51:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 11:51:02 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: Improve the repr
Message-ID: <20131022095102.9BFE31C30E8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67496:a9b51ff40cf6
Date: 2013-10-22 11:46 +0200
http://bitbucket.org/pypy/pypy/changeset/a9b51ff40cf6/

Log:	Improve the repr

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1715,6 +1715,10 @@
 
     def __str__(self):
         items = self.items
+        extra = ''
+        if self._TYPE._is_overallocated_array():
+            extra = ' allocated=%d' % len(items)
+            items = items[:self.used_len]
         if len(items) > 20:
             items = items[:12] + items[-5:]
             skipped_at = 12
@@ -1723,7 +1727,7 @@
         items = [self._str_item(item) for item in items]
         if skipped_at:
             items.insert(skipped_at, '(...)')
-        return 'array [ %s ]' % (', '.join(items),)
+        return 'array%s [ %s ]' % (extra, ', '.join(items))
 
     def getlength(self):
         return len(self.items)
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -824,6 +824,9 @@
     py.test.raises(IndexError, "a[3] = 43")
     a.used_length = 5
     py.test.raises(UninitializedMemoryAccess, "a[3]")
+    a[0] = 100
+    a[3] = 200
+    assert repr(a) == '<* array allocated=10 [ 100, #, #, 200, # ]>'
 
 class TestTrackAllocation:
     def test_automatic_tracking(self):

From noreply at buildbot.pypy.org  Tue Oct 22 11:51:04 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 11:51:04 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: (fijal,
	arigo)  _example()
Message-ID: <20131022095104.2C89B1C30E8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67497:c2edf6cd836f
Date: 2013-10-22 11:50 +0200
http://bitbucket.org/pypy/pypy/changeset/c2edf6cd836f/

Log:	(fijal, arigo) _example()

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1682,7 +1682,10 @@
             raise ValueError, "negative array length"
         _parentable.__init__(self, TYPE)
         if TYPE._is_overallocated_array():
-            self.used_len = 0
+            if initialization == 'example':
+                self.used_len = n
+            else:
+                self.used_len = 0
         myrange = self._check_range(n)
         self.items = [TYPE.OF._allocate(initialization=initialization,
                                         parent=self, parentindex=j)
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -827,6 +827,9 @@
     a[0] = 100
     a[3] = 200
     assert repr(a) == '<* array allocated=10 [ 100, #, #, 200, # ]>'
+    #
+    b = Ptr(A)._example()
+    assert repr(b) == '<* array allocated=1 [ 0 ]>'
 
 class TestTrackAllocation:
     def test_automatic_tracking(self):

From noreply at buildbot.pypy.org  Tue Oct 22 12:05:13 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 12:05:13 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: (fijal around,
	arigo)
Message-ID: <20131022100513.E96721D236E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67498:75ab419638af
Date: 2013-10-22 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/75ab419638af/

Log:	(fijal around, arigo)

	Support rtyping the operations specific to overallocated arrays.

diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -624,6 +624,19 @@
         if ITEMTYPE is not lltype.Void:
             array[index] = item
 
+    def op_getarrayallocatedlength(self, obj):
+        checkptr(obj)
+        return obj.allocated_length
+
+    def op_getarrayusedlength(self, obj):
+        checkptr(obj)
+        return obj.used_length
+
+    def op_setarrayusedlength(self, obj, nlen):
+        checkptr(obj)
+        assert isinstance(nlen, int)
+        obj.used_length = nlen
+
     def perform_call(self, f, ARGS, args):
         fobj = self.llinterpreter.typer.type_system.deref(f)
         has_callable = getattr(fobj, '_callable', None) is not None
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -386,6 +386,9 @@
     'bare_setfield':        LLOp(),
     'setarrayitem':         LLOp(),
     'bare_setarrayitem':    LLOp(),
+    'getarrayallocatedlength': LLOp(sideeffects=False), # over-allocated arrays
+    'getarrayusedlength':   LLOp(sideeffects=False),    # over-allocated arrays
+    'setarrayusedlength':   LLOp(),                     # over-allocated arrays
     'cast_pointer':         LLOp(canfold=True),
     'ptr_eq':               LLOp(canfold=True),
     'ptr_ne':               LLOp(canfold=True),
diff --git a/rpython/rtyper/rptr.py b/rpython/rtyper/rptr.py
--- a/rpython/rtyper/rptr.py
+++ b/rpython/rtyper/rptr.py
@@ -47,6 +47,16 @@
         else:
             assert hop.s_result.is_constant()
             return hop.inputconst(hop.r_result, hop.s_result.const)
+        if self.lowleveltype.TO._is_overallocated_array():
+            v_self = hop.inputarg(self, arg=0)
+            if attr == 'allocated_length':
+                return hop.genop('getarrayallocatedlength', [v_self],
+                                 resulttype = lltype.Signed)
+            elif attr == 'used_length':
+                return hop.genop('getarrayusedlength', [v_self],
+                                 resulttype = lltype.Signed)
+            else:
+                raise TyperError("getattr(overallocated_array, %r)" % (attr,))
         assert attr in self.lowleveltype.TO._flds # check that the field exists
         FIELD_TYPE = getattr(self.lowleveltype.TO, attr)
         if isinstance(FIELD_TYPE, lltype.ContainerType):
@@ -65,6 +75,14 @@
 
     def rtype_setattr(self, hop):
         attr = hop.args_s[1].const
+        if self.lowleveltype.TO._is_overallocated_array():
+            if attr == 'used_length':
+                v_self = hop.inputarg(self, arg=0)
+                v_length = hop.inputarg(lltype.Signed, arg=2)
+                hop.genop('setarrayusedlength', [v_self, v_length])
+                return
+            else:
+                raise TyperError("setattr(overallocated_array, %r)" % (attr,))
         FIELD_TYPE = getattr(self.lowleveltype.TO, attr)
         assert not isinstance(FIELD_TYPE, lltype.ContainerType)
         vlist = hop.inputargs(self, lltype.Void, hop.args_r[2])
diff --git a/rpython/rtyper/test/test_llann.py b/rpython/rtyper/test/test_llann.py
--- a/rpython/rtyper/test/test_llann.py
+++ b/rpython/rtyper/test/test_llann.py
@@ -509,3 +509,18 @@
     assert res is True
     res = interpret(f, [25, 10])
     assert res is True
+
+
+def test_overallocated_array():
+    A = GcArray(Signed, hints={'overallocated': True})
+
+    def f():
+        a = malloc(A, 10)
+        a.used_length = 5
+        a[3] = 42
+        assert a[3] == 42
+        return a.used_length + (a.allocated_length * 100)
+
+    assert f() == 1005
+    res = interpret(f, [])
+    assert res == 1005

From noreply at buildbot.pypy.org  Tue Oct 22 12:07:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 12:07:00 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: Check that we
 cannot use len() on overallocated arrays
Message-ID: <20131022100700.15CF21D236F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67499:af232e633fbd
Date: 2013-10-22 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/af232e633fbd/

Log:	Check that we cannot use len() on overallocated arrays

diff --git a/rpython/rtyper/test/test_llann.py b/rpython/rtyper/test/test_llann.py
--- a/rpython/rtyper/test/test_llann.py
+++ b/rpython/rtyper/test/test_llann.py
@@ -404,6 +404,14 @@
         s = self.annotate(llf, [])
         assert s.is_constant() and s.const == 12
 
+    def test_cannot_use_len_on_overallocated_array(self):
+        A = GcArray(Signed, hints={'overallocated': True})
+        def llf():
+            a = malloc(A, 10)
+            return len(a)
+        e = py.test.raises(TypeError, self.annotate, llf, [])
+        assert str(e.value).endswith('has no length attribute')
+
 
 def test_pseudohighlevelcallable():
     t = TranslationContext()

From noreply at buildbot.pypy.org  Tue Oct 22 12:10:42 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 12:10:42 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: Another passing
	test
Message-ID: <20131022101043.014FD1C0144@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67500:ba49d53368d9
Date: 2013-10-22 12:09 +0200
http://bitbucket.org/pypy/pypy/changeset/ba49d53368d9/

Log:	Another passing test

diff --git a/rpython/rtyper/test/test_llann.py b/rpython/rtyper/test/test_llann.py
--- a/rpython/rtyper/test/test_llann.py
+++ b/rpython/rtyper/test/test_llann.py
@@ -532,3 +532,13 @@
     assert f() == 1005
     res = interpret(f, [])
     assert res == 1005
+
+def test_check_used_length_not_too_big():
+    A = GcArray(Signed, hints={'overallocated': True})
+
+    def f(n):
+        a = malloc(A, n)
+        a.used_length = 10
+
+    py.test.raises(ValueError, f, 5)
+    py.test.raises(ValueError, interpret, f, [5])

From noreply at buildbot.pypy.org  Tue Oct 22 12:14:46 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 12:14:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: Kill dead code
Message-ID: <20131022101446.1B2A61C0144@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67501:3e9f1cd83031
Date: 2013-10-22 12:14 +0200
http://bitbucket.org/pypy/pypy/changeset/3e9f1cd83031/

Log:	Kill dead code

diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -182,21 +182,6 @@
                                  FIELD_T):
                 yield line
 
-    def debug_offsets(self):
-        # generate number exprs giving the offset of the elements in the struct
-        assert self.varlength is None
-        for name in self.fieldnames:
-            FIELD_T = self.c_struct_field_type(name)
-            if FIELD_T is Void:
-                yield '-1'
-            else:
-                try:
-                    cname = self.c_struct_field_name(name)
-                except ValueError:
-                    yield '-1'
-                else:
-                    yield 'offsetof(%s %s, %s)' % (self.typetag,
-                                                   self.name, cname)
 
 def deflength(varlength):
     if varlength is None:
@@ -318,20 +303,6 @@
             yield '\t}'
             yield '}'
 
-    def debug_offsets(self):
-        # generate three offsets for debugging inspection
-        assert self.varlength is None
-        if not self.ARRAY._hints.get('nolength', False):
-            yield 'offsetof(struct %s, length)' % (self.name,)
-        else:
-            yield '-1'
-        if self.ARRAY.OF is not Void:
-            yield 'offsetof(struct %s, items[0])' % (self.name,)
-            yield 'offsetof(struct %s, items[1])' % (self.name,)
-        else:
-            yield '-1'
-            yield '-1'
-
 
 class BareBoneArrayDefNode(NodeWithDependencies):
     """For 'simple' array types which don't need a length nor GC headers.
@@ -391,12 +362,6 @@
     def visitor_lines(self, prefix, on_item):
         raise Exception("cannot visit C arrays - don't know the length")
 
-    def debug_offsets(self):
-        # generate three offsets for debugging inspection,
-        yield '-1'     # no length
-        yield '0'      # first element is immediately at the start of the array
-        yield 'sizeof(%s)' % (cdecl(self.itemtypename, ''),)
-
 
 class FixedSizeArrayDefNode(NodeWithDependencies):
     gcinfo = None
@@ -469,10 +434,6 @@
             yield '\t}'
             yield '}'
 
-    def debug_offsets(self):
-        # XXX not implemented
-        return []
-
 
 class ExtTypeOpaqueDefNode(NodeWithDependencies):
     """For OpaqueTypes created with the hint render_structure."""

From noreply at buildbot.pypy.org  Tue Oct 22 12:16:21 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 12:16:21 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: hg merge default
Message-ID: <20131022101621.A43551C0144@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67502:5d3e8631c3fe
Date: 2013-10-22 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/5d3e8631c3fe/

Log:	hg merge default

diff too long, truncating to 2000 out of 4811 lines

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,51 +1,17 @@
-import core
-from core import *
-import lib
-from lib import *
+from . import core
+from .core import *
+from . import lib
+from .lib import *
 
 from __builtin__ import bool, int, long, float, complex, object, unicode, str
-from core import abs, max, min
+
+from .core import round, abs, max, min
 
 __version__ = '1.7.0'
 
-import os
-def get_include():
-    """
-    Return the directory that contains the NumPy \\*.h header files.
-
-    Extension modules that need to compile against NumPy should use this
-    function to locate the appropriate include directory.
-
-    Notes
-    -----
-    When using ``distutils``, for example in ``setup.py``.
-    ::
-
-        import numpy as np
-        ...
-        Extension('extension_name', ...
-                include_dirs=[np.get_include()])
-        ...
-
-    """
-    try:
-        import numpy
-    except:
-        # running from pypy source directory
-        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-        return os.path.join(head, '../include')
-    else:
-        # using installed numpy core headers
-        import numpy.core as core
-        d = os.path.join(os.path.dirname(core.__file__), 'include')
-    return d
-
-
-
-__all__ = ['__version__', 'get_include']
+__all__ = ['__version__']
 __all__ += core.__all__
 __all__ += lib.__all__
+
 #import sys
 #sys.modules.setdefault('numpy', sys.modules['numpypy'])
-
-
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ b/lib_pypy/numpypy/core/arrayprint.py
@@ -247,10 +247,11 @@
     formatdict = {'bool' : _boolFormatter,
                   'int' : IntegerFormat(data),
                   'float' : FloatFormat(data, precision, suppress_small),
-                  'longfloat' : LongFloatFormat(precision),
+                  'longfloat' : FloatFormat(data, precision, suppress_small),
                   'complexfloat' : ComplexFormat(data, precision,
                                                  suppress_small),
-                  'longcomplexfloat' : LongComplexFormat(precision),
+                  'longcomplexfloat' : ComplexFormat(data, precision,
+                                                     suppress_small),
                   'datetime' : DatetimeFormat(data),
                   'timedelta' : TimedeltaFormat(data),
                   'numpystr' : repr_format,
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -1,12 +1,13 @@
 from __future__ import division, absolute_import, print_function
 
 __all__ = [
-           'newaxis', 'ufunc',
+           'newaxis', 'ufunc', 'argwhere',
            'asarray', 'asanyarray', 'base_repr',
            'array_repr', 'array_str', 'set_string_function',
            'array_equal', 'array_equiv', 'outer', 'vdot', 'identity', 'little_endian',
-           'Inf', 'inf', 'infty', 'Infinity', 'nan', 'NaN', 'False_', 'True_',
-           'seterr',
+           'seterr', 'flatnonzero',
+           'Inf', 'inf', 'infty', 'Infinity',
+           'nan', 'NaN', 'False_', 'True_',
           ]
 
 import sys
@@ -165,6 +166,85 @@
     """
     return array(a, dtype, copy=False, order=order, subok=True)
 
+def argwhere(a):
+    """
+    Find the indices of array elements that are non-zero, grouped by element.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+
+    Returns
+    -------
+    index_array : ndarray
+        Indices of elements that are non-zero. Indices are grouped by element.
+
+    See Also
+    --------
+    where, nonzero
+
+    Notes
+    -----
+    ``np.argwhere(a)`` is the same as ``np.transpose(np.nonzero(a))``.
+
+    The output of ``argwhere`` is not suitable for indexing arrays.
+    For this purpose use ``where(a)`` instead.
+
+    Examples
+    --------
+    >>> x = np.arange(6).reshape(2,3)
+    >>> x
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.argwhere(x>1)
+    array([[0, 2],
+           [1, 0],
+           [1, 1],
+           [1, 2]])
+
+    """
+    return transpose(asanyarray(a).nonzero())
+
+def flatnonzero(a):
+    """
+    Return indices that are non-zero in the flattened version of a.
+
+    This is equivalent to a.ravel().nonzero()[0].
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array.
+
+    Returns
+    -------
+    res : ndarray
+        Output array, containing the indices of the elements of `a.ravel()`
+        that are non-zero.
+
+    See Also
+    --------
+    nonzero : Return the indices of the non-zero elements of the input array.
+    ravel : Return a 1-D array containing the elements of the input array.
+
+    Examples
+    --------
+    >>> x = np.arange(-2, 3)
+    >>> x
+    array([-2, -1,  0,  1,  2])
+    >>> np.flatnonzero(x)
+    array([0, 1, 3, 4])
+
+    Use the indices of the non-zero elements as an index array to extract
+    these elements:
+
+    >>> x.ravel()[np.flatnonzero(x)]
+    array([-2, -1,  1,  2])
+
+    """
+    return a.ravel().nonzero()[0]
+
 def base_repr(number, base=2, padding=0):
     """
     Return a string representation of a number in the given base system.
diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -395,6 +395,9 @@
                   ('int_', 'long'),
                   ('uint', 'ulong'),
                   ('cfloat', 'cdouble'),
+                  ('longfloat', 'longdouble'),
+                  ('clongfloat', 'clongdouble'),
+                  ('longcomplex', 'clongdouble'),
                   ('bool_', 'bool'),
                   ('unicode_', 'unicode'),
                   ]
diff --git a/lib_pypy/numpypy/lib/__init__.py b/lib_pypy/numpypy/lib/__init__.py
--- a/lib_pypy/numpypy/lib/__init__.py
+++ b/lib_pypy/numpypy/lib/__init__.py
@@ -1,11 +1,16 @@
-import function_base
-from function_base import *
-import shape_base
-from shape_base import *
-import twodim_base
-from twodim_base import *
+from __future__ import division, absolute_import, print_function
 
-__all__ = []
+import math
+
+from .function_base import *
+from .shape_base import *
+from .twodim_base import *
+from .ufunclike import *
+from .utils import *
+
+__all__ = ['math']
 __all__ += function_base.__all__
 __all__ += shape_base.__all__
 __all__ += twodim_base.__all__
+__all__ += ufunclike.__all__
+__all__ += utils.__all__
diff --git a/lib_pypy/numpypy/lib/ufunclike.py b/lib_pypy/numpypy/lib/ufunclike.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/lib/ufunclike.py
@@ -0,0 +1,177 @@
+"""
+Module of functions that are like ufuncs in acting on arrays and optionally
+storing results in an output array.
+
+"""
+from __future__ import division, absolute_import, print_function
+
+__all__ = ['fix', 'isneginf', 'isposinf']
+
+from ..core import numeric as nx
+
+def fix(x, y=None):
+    """
+    Round to nearest integer towards zero.
+
+    Round an array of floats element-wise to nearest integer towards zero.
+    The rounded values are returned as floats.
+
+    Parameters
+    ----------
+    x : array_like
+        An array of floats to be rounded
+    y : ndarray, optional
+        Output array
+
+    Returns
+    -------
+    out : ndarray of floats
+        The array of rounded numbers
+
+    See Also
+    --------
+    trunc, floor, ceil
+    around : Round to given number of decimals
+
+    Examples
+    --------
+    >>> np.fix(3.14)
+    3.0
+    >>> np.fix(3)
+    3.0
+    >>> np.fix([2.1, 2.9, -2.1, -2.9])
+    array([ 2.,  2., -2., -2.])
+
+    """
+    x = nx.asanyarray(x)
+    y1 = nx.floor(x)
+    y2 = nx.ceil(x)
+    if y is None:
+        y = nx.asanyarray(y1)
+    y[...] = nx.where(x >= 0, y1, y2)
+    return y
+
+def isposinf(x, y=None):
+    """
+    Test element-wise for positive infinity, return result as bool array.
+
+    Parameters
+    ----------
+    x : array_like
+        The input array.
+    y : array_like, optional
+        A boolean array with the same shape as `x` to store the result.
+
+    Returns
+    -------
+    y : ndarray
+        A boolean array with the same dimensions as the input.
+        If second argument is not supplied then a boolean array is returned
+        with values True where the corresponding element of the input is
+        positive infinity and values False where the element of the input is
+        not positive infinity.
+
+        If a second argument is supplied the result is stored there. If the
+        type of that array is a numeric type the result is represented as zeros
+        and ones, if the type is boolean then as False and True.
+        The return value `y` is then a reference to that array.
+
+    See Also
+    --------
+    isinf, isneginf, isfinite, isnan
+
+    Notes
+    -----
+    Numpy uses the IEEE Standard for Binary Floating-Point for Arithmetic
+    (IEEE 754).
+
+    Errors result if the second argument is also supplied when `x` is a
+    scalar input, or if first and second arguments have different shapes.
+
+    Examples
+    --------
+    >>> np.isposinf(np.PINF)
+    array(True, dtype=bool)
+    >>> np.isposinf(np.inf)
+    array(True, dtype=bool)
+    >>> np.isposinf(np.NINF)
+    array(False, dtype=bool)
+    >>> np.isposinf([-np.inf, 0., np.inf])
+    array([False, False,  True], dtype=bool)
+
+    >>> x = np.array([-np.inf, 0., np.inf])
+    >>> y = np.array([2, 2, 2])
+    >>> np.isposinf(x, y)
+    array([0, 0, 1])
+    >>> y
+    array([0, 0, 1])
+
+    """
+    if y is None:
+        x = nx.asarray(x)
+        y = nx.empty(x.shape, dtype=nx.bool_)
+    nx.logical_and(nx.isinf(x), ~nx.signbit(x), y)
+    return y
+
+def isneginf(x, y=None):
+    """
+    Test element-wise for negative infinity, return result as bool array.
+
+    Parameters
+    ----------
+    x : array_like
+        The input array.
+    y : array_like, optional
+        A boolean array with the same shape and type as `x` to store the
+        result.
+
+    Returns
+    -------
+    y : ndarray
+        A boolean array with the same dimensions as the input.
+        If second argument is not supplied then a numpy boolean array is
+        returned with values True where the corresponding element of the
+        input is negative infinity and values False where the element of
+        the input is not negative infinity.
+
+        If a second argument is supplied the result is stored there. If the
+        type of that array is a numeric type the result is represented as
+        zeros and ones, if the type is boolean then as False and True. The
+        return value `y` is then a reference to that array.
+
+    See Also
+    --------
+    isinf, isposinf, isnan, isfinite
+
+    Notes
+    -----
+    Numpy uses the IEEE Standard for Binary Floating-Point for Arithmetic
+    (IEEE 754).
+
+    Errors result if the second argument is also supplied when x is a scalar
+    input, or if first and second arguments have different shapes.
+
+    Examples
+    --------
+    >>> np.isneginf(np.NINF)
+    array(True, dtype=bool)
+    >>> np.isneginf(np.inf)
+    array(False, dtype=bool)
+    >>> np.isneginf(np.PINF)
+    array(False, dtype=bool)
+    >>> np.isneginf([-np.inf, 0., np.inf])
+    array([ True, False, False], dtype=bool)
+
+    >>> x = np.array([-np.inf, 0., np.inf])
+    >>> y = np.array([2, 2, 2])
+    >>> np.isneginf(x, y)
+    array([1, 0, 0])
+    >>> y
+    array([1, 0, 0])
+
+    """
+    if y is None:
+        x = nx.asarray(x)
+        y = nx.empty(x.shape, dtype=nx.bool_)
+    nx.logical_and(nx.isinf(x), nx.signbit(x), y)
+    return y
diff --git a/lib_pypy/numpypy/lib/utils.py b/lib_pypy/numpypy/lib/utils.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/lib/utils.py
@@ -0,0 +1,34 @@
+import os
+
+__all__ = ['get_include']
+
+def get_include():
+    """
+    Return the directory that contains the NumPy \\*.h header files.
+
+    Extension modules that need to compile against NumPy should use this
+    function to locate the appropriate include directory.
+
+    Notes
+    -----
+    When using ``distutils``, for example in ``setup.py``.
+    ::
+
+        import numpy as np
+        ...
+        Extension('extension_name', ...
+                include_dirs=[np.get_include()])
+        ...
+
+    """
+    try:
+        import numpy
+    except:
+        # running from pypy source directory
+        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
+        return os.path.join(head, '../include')
+    else:
+        # using installed numpy core headers
+        import numpy.core as core
+        d = os.path.join(os.path.dirname(core.__file__), 'include')
+    return d
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -109,4 +109,5 @@
 .. branch: file-support-in-rpython
 make open() and friends rpython
 
-
+.. branch: incremental-gc
+Added the new incminimark GC which performs GC in incremental steps
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -239,6 +239,18 @@
         # _____ this code is here to support testing only _____
         return self
 
+    def unpackiterable_int(self, space):
+        lst = space.listview_int(self)
+        if lst:
+            return lst[:]
+        return None
+
+    def unpackiterable_float(self, space):
+        lst = space.listview_float(self)
+        if lst:
+            return lst[:]
+        return None
+
 
 class W_InterpIterable(W_Root):
     def __init__(self, space, w_iterable):
@@ -838,6 +850,22 @@
         return self._unpackiterable_known_length_jitlook(w_iterator,
                                                          expected_length)
 
+
+    def unpackiterable_int(self, w_obj):
+        """
+        Return a RPython list of unwrapped ints out of w_obj. The list is
+        guaranteed to be acopy of the actual data contained in w_obj, so you
+        can freely modify it. It might return None if not supported.
+        """
+        return w_obj.unpackiterable_int(self)
+
+    def unpackiterable_float(self, w_obj):
+        """
+        Same as unpackiterable_int, but for floats.
+        """
+        return w_obj.unpackiterable_float(self)
+
+
     def length_hint(self, w_obj, default):
         """Return the length of an object, consulting its __length_hint__
         method if necessary.
@@ -895,6 +923,20 @@
         """
         return None
 
+    def listview_int(self, w_list):
+        """ Return a list of unwrapped int out of a list of int. If the
+        argument is not a list or does not contain only int, return None.
+        May return None anyway.
+        """
+        return None
+
+    def listview_float(self, w_list):
+        """ Return a list of unwrapped float out of a list of float. If the
+        argument is not a list or does not contain only float, return None.
+        May return None anyway.
+        """
+        return None
+
     def view_as_kwargs(self, w_dict):
         """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
         strings and one of wrapped values. otherwise return (None, None)
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -282,6 +282,12 @@
     def iter(self):
         return self.ctype.iter(self)
 
+    def unpackiterable_int(self, space):
+        return self.ctype.aslist_int(self)
+
+    def unpackiterable_float(self, space):
+        return self.ctype.aslist_float(self)
+
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
         misc.write_raw_signed_data(self._cdata, source, self.ctype.size)
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -105,6 +105,26 @@
     def iter(self, cdata):
         return W_CDataIter(self.space, self.ctitem, cdata)
 
+    def aslist_int(self, cdata):
+        from rpython.rlib.rarray import populate_list_from_raw_array
+        if self.ctitem.is_long():
+            res = []
+            buf = rffi.cast(rffi.LONGP, cdata._cdata)
+            length = cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        return None
+
+    def aslist_float(self, cdata):
+        from rpython.rlib.rarray import populate_list_from_raw_array
+        if self.ctitem.is_double():
+            res = []
+            buf = rffi.cast(rffi.DOUBLEP, cdata._cdata)
+            length = cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        return None
+
     def get_vararg_type(self):
         return self.ctptr
 
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -43,6 +43,12 @@
     def is_unichar_ptr_or_array(self):
         return False
 
+    def is_long(self):
+        return False
+
+    def is_double(self):
+        return False
+
     def newp(self, w_init):
         space = self.space
         raise operationerrfmt(space.w_TypeError,
@@ -163,6 +169,9 @@
                               "cdata '%s' does not support iteration",
                               self.name)
 
+    def unpackiterable_int(self, cdata):
+        return None
+
     def get_vararg_type(self):
         return self
 
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -85,7 +85,6 @@
             return self.space.wrap(s)
         return W_CType.string(self, cdataobj, maxlen)
 
-
 class W_CTypePrimitiveCharOrUniChar(W_CTypePrimitive):
     _attrs_ = []
     is_primitive_integer = True
@@ -171,6 +170,9 @@
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
 
+    def is_long(self):
+        return self.size == rffi.sizeof(lltype.Signed)
+
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
 
@@ -274,6 +276,9 @@
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
 
+    def is_double(self):
+        return self.size == rffi.sizeof(lltype.Float)
+
     def cast(self, w_ob):
         space = self.space
         if isinstance(w_ob, cdataobj.W_CData):
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -42,6 +42,12 @@
     def is_char_or_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveCharOrUniChar)
 
+    def aslist_int(self, cdata):
+        return None
+
+    def aslist_float(self, cdata):
+        return None
+
     def cast(self, w_ob):
         # cast to a pointer, to a funcptr, or to an array.
         # Note that casting to an array is an extension to the C language,
@@ -58,19 +64,45 @@
             value = rffi.cast(rffi.CCHARP, value)
         return cdataobj.W_CData(space, value, self)
 
+    def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
+        from rpython.rlib.rarray import copy_list_to_raw_array
+        int_list = self.space.listview_int(w_ob)
+        float_list = self.space.listview_float(w_ob)
+        #
+        if self.ctitem.is_long() and int_list is not None:
+            cdata = rffi.cast(rffi.LONGP, cdata)
+            copy_list_to_raw_array(int_list, cdata)
+            return True
+        #
+        if self.ctitem.is_double() and float_list is not None:
+            cdata = rffi.cast(rffi.DOUBLEP, cdata)
+            copy_list_to_raw_array(float_list, cdata)
+            return True
+        #
+        return False
+
+    def _convert_array_from_listview(self, cdata, w_ob):
+        space = self.space
+        lst_w = space.listview(w_ob)
+        if self.length >= 0 and len(lst_w) > self.length:
+            raise operationerrfmt(space.w_IndexError,
+                "too many initializers for '%s' (got %d)",
+                                  self.name, len(lst_w))
+        ctitem = self.ctitem
+        for i in range(len(lst_w)):
+            ctitem.convert_from_object(cdata, lst_w[i])
+            cdata = rffi.ptradd(cdata, ctitem.size)
+
     def convert_array_from_object(self, cdata, w_ob):
         space = self.space
+        if self._convert_array_from_list_strategy_maybe(cdata, w_ob):
+            # the fast path worked, we are done now
+            return
+        #
+        # continue with the slow path
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
-            lst_w = space.listview(w_ob)
-            if self.length >= 0 and len(lst_w) > self.length:
-                raise operationerrfmt(space.w_IndexError,
-                    "too many initializers for '%s' (got %d)",
-                                      self.name, len(lst_w))
-            ctitem = self.ctitem
-            for i in range(len(lst_w)):
-                ctitem.convert_from_object(cdata, lst_w[i])
-                cdata = rffi.ptradd(cdata, ctitem.size)
+            self._convert_array_from_listview(cdata, w_ob)
         elif (self.can_cast_anything or
               (self.ctitem.is_primitive_integer and
                self.ctitem.size == rffi.sizeof(lltype.Char))):
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -0,0 +1,100 @@
+# side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule()
+from pypy.module._cffi_backend import misc
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+
+class AppTest_fast_path_from_list(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def setup_method(self, meth):
+        def forbidden(self, *args):
+            assert False, 'The slow path is forbidden'
+        self._original = W_CTypePtrOrArray._convert_array_from_listview.im_func
+        W_CTypePtrOrArray._convert_array_from_listview = forbidden
+
+    def teardown_method(self, meth):
+        W_CTypePtrOrArray._convert_array_from_listview = self._original
+
+    def test_fast_init_from_list(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, None)
+        buf = _cffi_backend.newp(LONG_ARRAY, [1, 2, 3])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == 3
+
+    def test_fast_init_from_list_float(self):
+        import _cffi_backend
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        P_DOUBLE = _cffi_backend.new_pointer_type(DOUBLE)
+        DOUBLE_ARRAY = _cffi_backend.new_array_type(P_DOUBLE, None)
+        buf = _cffi_backend.newp(DOUBLE_ARRAY, [1.1, 2.2, 3.3])
+        assert buf[0] == 1.1
+        assert buf[1] == 2.2
+        assert buf[2] == 3.3
+
+
+class AppTest_fast_path_to_list(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def setup_method(self, meth):
+        from pypy.interpreter import gateway
+        from rpython.rlib import rarray
+        #
+        self.count = 0
+        def get_count(*args):
+            return self.space.wrap(self.count)
+        self.w_get_count = self.space.wrap(gateway.interp2app(get_count))
+        #
+        original = rarray.populate_list_from_raw_array
+        def populate_list_from_raw_array(*args):
+            self.count += 1
+            return original(*args)
+        self._original = original
+        rarray.populate_list_from_raw_array = populate_list_from_raw_array
+        #
+        self.w_runappdirect = self.space.wrap(self.runappdirect)
+
+
+    def teardown_method(self, meth):
+        from rpython.rlib import rarray
+        rarray.populate_list_from_raw_array = self._original
+
+    def test_list_int(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, 3)
+        buf = _cffi_backend.newp(LONG_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_TypeError_if_no_length(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, 3)
+        buf = _cffi_backend.newp(LONG_ARRAY)
+        pbuf = _cffi_backend.cast(P_LONG, buf)
+        raises(TypeError, "list(pbuf)")
+
+
+    def test_list_float(self):
+        import _cffi_backend
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        P_DOUBLE = _cffi_backend.new_pointer_type(DOUBLE)
+        DOUBLE_ARRAY = _cffi_backend.new_array_type(P_DOUBLE, 3)
+        buf = _cffi_backend.newp(DOUBLE_ARRAY)
+        buf[0] = 1.1
+        buf[1] = 2.2
+        buf[2] = 3.3
+        lst = list(buf)
+        assert lst == [1.1, 2.2, 3.3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -1,5 +1,4 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module.micronumpy.interp_boxes import long_double_size, ENABLED_LONG_DOUBLE
 
 
 class MultiArrayModule(MixedModule):
@@ -64,6 +63,7 @@
         ("less_equal", "less_equal"),
         ("maximum", "maximum"),
         ("minimum", "minimum"),
+        ("mod", "mod"),
         ("multiply", "multiply"),
         ("negative", "negative"),
         ("not_equal", "not_equal"),
@@ -91,8 +91,6 @@
         ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
-        ('isneginf', 'isneginf'),
-        ('isposinf', 'isposinf'),
         ('isfinite', 'isfinite'),
         ('logical_and', 'logical_and'),
         ('logical_xor', 'logical_xor'),
diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -102,13 +102,10 @@
             dtype =  self.dtype.float_type
             return SliceArray(self.start + dtype.get_size(), strides,
                     backstrides, self.get_shape(), self, orig_array, dtype=dtype)
-        if self.dtype.is_flexible_type():
-            # numpy returns self for self.imag
-            return SliceArray(self.start, strides, backstrides,
-                    self.get_shape(), self, orig_array)
         impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides,
                              backstrides)
-        impl.fill(self.dtype.box(0))
+        if not self.dtype.is_flexible_type():
+            impl.fill(self.dtype.box(0))
         return impl
 
     def set_imag(self, space, orig_array, w_value):
@@ -129,7 +126,8 @@
                 idx = self.get_shape()[i] + idx
             if idx < 0 or idx >= self.get_shape()[i]:
                 raise operationerrfmt(space.w_IndexError,
-                      "index (%d) out of range (0<=index<%d", i, self.get_shape()[i],
+                      "index %d is out of bounds for axis %d with size %d",
+                      idx, i, self.get_shape()[i],
                 )
             item += idx * strides[i]
         return item
@@ -145,7 +143,8 @@
                 idx = shape[i] + idx
             if idx < 0 or idx >= shape[i]:
                 raise operationerrfmt(space.w_IndexError,
-                      "index (%d) out of range (0<=index<%d", i, shape[i],
+                      "index %d is out of bounds for axis %d with size %d",
+                      idx, i, self.get_shape()[i],
                 )
             item += idx * strides[i]
         return item
@@ -380,8 +379,8 @@
 
 class NonWritableArray(ConcreteArray):
     def descr_setitem(self, space, orig_array, w_index, w_value):
-        raise OperationError(space.w_RuntimeError, space.wrap(
-            "array is not writable"))
+        raise OperationError(space.w_ValueError, space.wrap(
+            "assignment destination is read-only"))
 
 
 class SliceArray(BaseConcreteArray):
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -8,7 +8,7 @@
 from rpython.rlib.rawstorage import raw_storage_getitem, raw_storage_setitem, \
         free_raw_storage, alloc_raw_storage
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import widen
 from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError
 from pypy.module.micronumpy.base import W_NDimArray
@@ -43,7 +43,7 @@
                                     + self.start + step * i)
                     v.append(_v)
             if comp_type == 'int':
-                v = intmask(v)
+                v = widen(v)
             elif comp_type == 'float':
                 v = float(v)
             elif comp_type == 'complex':
@@ -100,10 +100,15 @@
     if count < 2:
         def arg_lt(a, b):
             # Does numpy do <= ?
-            return a[0] < b[0]
+            return a[0] < b[0] or b[0] != b[0] and a[0] == a[0]
     else:
         def arg_lt(a, b):
             for i in range(count):
+                if b[0][i] != b[0][i] and a[0][i] == a[0][i]:
+                    return True
+                elif b[0][i] == b[0][i] and a[0][i] != a[0][i]:
+                    return False
+            for i in range(count):
                 if a[0][i] < b[0][i]:
                     return True
                 elif a[0][i] > b[0][i]:
@@ -200,7 +205,7 @@
                                     + self.start + step * i)
                     v.append(_v)
             if comp_type == 'int':
-                v = intmask(v)
+                v = widen(v)
             elif comp_type == 'float':
                 v = float(v)
             elif comp_type == 'complex':
@@ -318,7 +323,8 @@
 
 all_types = (types.all_float_types + types.all_complex_types +
              types.all_int_types)
-all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
+all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__ and
+                                     not issubclass(i[0], types.BaseFloat16)]
 all_types = unrolling_iterable(all_types)
 
 class ArgSortCache(object):
diff --git a/pypy/module/micronumpy/constants.py b/pypy/module/micronumpy/constants.py
--- a/pypy/module/micronumpy/constants.py
+++ b/pypy/module/micronumpy/constants.py
@@ -1,4 +1,21 @@
+from pypy.interpreter.error import OperationError
 
-MODE_WRAP, MODE_RAISE, MODE_CLIP = range(3)
+MODE_CLIP, MODE_WRAP, MODE_RAISE = range(3)
 
-MODES = {'wrap': MODE_WRAP, 'raise': MODE_RAISE, 'clip': MODE_CLIP}
+def clipmode_converter(space, w_mode):
+    if space.is_none(w_mode):
+        return MODE_RAISE
+    if space.isinstance_w(w_mode, space.w_str):
+        mode = space.str_w(w_mode)
+        if mode.startswith('C') or mode.startswith('c'):
+            return MODE_CLIP
+        if mode.startswith('W') or mode.startswith('w'):
+            return MODE_WRAP
+        if mode.startswith('R') or mode.startswith('r'):
+            return MODE_RAISE
+    elif space.isinstance_w(w_mode, space.w_int):
+        mode = space.int_w(w_mode)
+        if MODE_CLIP <= mode <= MODE_RAISE:
+            return mode
+    raise OperationError(space.w_TypeError,
+                         space.wrap("clipmode not understood"))
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -1,10 +1,9 @@
-
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
-from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs
+from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs, constants
 from pypy.module.micronumpy.iter import Chunk, Chunks
 from pypy.module.micronumpy.strides import shape_agreement,\
      shape_agreement_multiple
-from pypy.module.micronumpy.constants import MODES
+from pypy.module.micronumpy.constants import clipmode_converter
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
 
@@ -118,12 +117,12 @@
                 shape[i] += axis_size
         a_dt = arr.get_dtype()
         if dtype.is_record_type() and a_dt.is_record_type():
-            #Record types must match
+            # Record types must match
             for f in dtype.fields:
                 if f not in a_dt.fields or \
                              dtype.fields[f] != a_dt.fields[f]:
                     raise OperationError(space.w_TypeError,
-                               space.wrap("record type mismatch"))
+                               space.wrap("invalid type promotion"))
         elif dtype.is_record_type() or a_dt.is_record_type():
             raise OperationError(space.w_TypeError,
                         space.wrap("invalid type promotion"))
@@ -171,8 +170,7 @@
 def count_nonzero(space, w_obj):
     return space.wrap(loop.count_all_true(convert_to_array(space, w_obj)))
 
- at unwrap_spec(mode=str)
-def choose(space, w_arr, w_choices, w_out, mode):
+def choose(space, w_arr, w_choices, w_out, w_mode):
     arr = convert_to_array(space, w_arr)
     choices = [convert_to_array(space, w_item) for w_item
                in space.listview(w_choices)]
@@ -187,23 +185,16 @@
     shape = shape_agreement_multiple(space, choices + [w_out])
     out = interp_dtype.dtype_agreement(space, choices, shape, w_out)
     dtype = out.get_dtype()
-    if mode not in MODES:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("mode %s not known" % (mode,)))
-    loop.choose(space, arr, choices, shape, dtype, out, MODES[mode])
+    mode = clipmode_converter(space, w_mode)
+    loop.choose(space, arr, choices, shape, dtype, out, mode)
     return out
 
-
- at unwrap_spec(mode=str)
-def put(space, w_arr, w_indices, w_values, mode='raise'):
-    from pypy.module.micronumpy import constants
+def put(space, w_arr, w_indices, w_values, w_mode):
     from pypy.module.micronumpy.support import int_w
 
     arr = convert_to_array(space, w_arr)
+    mode = clipmode_converter(space, w_mode)
 
-    if mode not in constants.MODES:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("mode %s not known" % (mode,)))
     if not w_indices:
         raise OperationError(space.w_ValueError,
                              space.wrap("indice list cannot be empty"))
@@ -228,13 +219,13 @@
         index = int_w(space, idx)
 
         if index < 0 or index >= arr.get_size():
-            if constants.MODES[mode] == constants.MODE_RAISE:
-                raise OperationError(space.w_ValueError, space.wrap(
-                    "invalid entry in choice array"))
-            elif constants.MODES[mode] == constants.MODE_WRAP:
+            if mode == constants.MODE_RAISE:
+                raise OperationError(space.w_IndexError, space.wrap(
+                    "index %d is out of bounds for axis 0 with size %d" % (index, arr.get_size())))
+            elif mode == constants.MODE_WRAP:
                 index = index % arr.get_size()
             else:
-                assert constants.MODES[mode] == constants.MODE_CLIP
+                assert mode == constants.MODE_CLIP
                 if index < 0:
                     index = 0
                 else:
@@ -247,7 +238,6 @@
 
         arr.setitem(space, [index], dtype.coerce(space, value))
 
-
 def diagonal(space, arr, offset, axis1, axis2):
     shape = arr.get_shape()
     shapelen = len(shape)
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -20,14 +20,14 @@
 MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
 MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else ()
 
-# Is this the proper place for this?
-ENABLED_LONG_DOUBLE = False
-long_double_size = rffi.sizeof_c_type('long double', ignore_errors=True)
+#long_double_size = rffi.sizeof_c_type('long double', ignore_errors=True)
+#import os
+#if long_double_size == 8 and os.name == 'nt':
+#    # this is a lie, or maybe a wish, MS fakes longdouble math with double
+#    long_double_size = 12
 
-import os
-if long_double_size == 8 and os.name == 'nt':
-    # this is a lie, or maybe a wish, MS fakes longdouble math with double
-    long_double_size = 12
+# hardcode to 8 for now (simulate using normal double) until long double works
+long_double_size = 8
 
 
 def new_dtype_getter(name):
@@ -63,6 +63,7 @@
 
 class PrimitiveBox(Box):
     _mixin_ = True
+    _immutable_fields_ = ['value']
 
     def __init__(self, value):
         self.value = value
@@ -82,11 +83,11 @@
         ret = builder.build()
 
         lltype.free(value, flavor="raw")
-
         return ret
 
 class ComplexBox(Box):
     _mixin_ = True
+    _immutable_fields_ = ['real', 'imag']
 
     def __init__(self, real, imag=0.):
         self.real = real
@@ -111,11 +112,11 @@
         ret = builder.build()
 
         lltype.free(value, flavor="raw")
-
         return ret
 
+
 class W_GenericBox(W_Root):
-    _attrs_ = ()
+    _attrs_ = []
 
     def descr__new__(space, w_subtype, __args__):
         raise operationerrfmt(space.w_TypeError,
@@ -125,12 +126,21 @@
     def get_dtype(self, space):
         return self._get_dtype(space)
 
+    def item(self, space):
+        return self.get_dtype(space).itemtype.to_builtin_type(space, self)
+
     def descr_str(self, space):
         return space.wrap(self.get_dtype(space).itemtype.str_format(self))
 
     def descr_format(self, space, w_spec):
         return space.format(self.item(space), w_spec)
 
+    def descr_hash(self, space):
+        return space.hash(self.item(space))
+
+    def descr_index(self, space):
+        return space.index(self.item(space))
+
     def descr_int(self, space):
         box = self.convert_to(W_LongBox._get_dtype(space))
         assert isinstance(box, W_LongBox)
@@ -222,19 +232,13 @@
         w_remainder = self.descr_rmod(space, w_other)
         return space.newtuple([w_quotient, w_remainder])
 
-    def descr_hash(self, space):
-        return space.hash(self.item(space))
-
-    def item(self, space):
-        return self.get_dtype(space).itemtype.to_builtin_type(space, self)
-
     def descr_any(self, space):
         value = space.is_true(self)
-        return space.wrap(W_BoolBox(value))
+        return self.get_dtype(space).box(value)
 
     def descr_all(self, space):
         value = space.is_true(self)
-        return space.wrap(W_BoolBox(value))
+        return self.get_dtype(space).box(value)
 
     def descr_ravel(self, space):
         from pypy.module.micronumpy.base import convert_to_array
@@ -260,7 +264,7 @@
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
 class W_NumberBox(W_GenericBox):
-    _attrs_ = ()
+    pass
 
 class W_IntegerBox(W_NumberBox):
     def int_w(self, space):
@@ -309,10 +313,10 @@
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter('ulonglong')
 
 class W_InexactBox(W_NumberBox):
-    _attrs_ = ()
+    pass
 
 class W_FloatingBox(W_InexactBox):
-    _attrs_ = ()
+    pass
 
 class W_Float16Box(W_FloatingBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float16")
@@ -323,9 +327,43 @@
 class W_Float64Box(W_FloatingBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float64")
 
+class W_ComplexFloatingBox(W_InexactBox):
+    def descr_get_real(self, space):
+        dtype = self._COMPONENTS_BOX._get_dtype(space)
+        box = self.convert_real_to(dtype)
+        assert isinstance(box, self._COMPONENTS_BOX)
+        return space.wrap(box)
+
+    def descr_get_imag(self, space):
+        dtype = self._COMPONENTS_BOX._get_dtype(space)
+        box = self.convert_imag_to(dtype)
+        assert isinstance(box, self._COMPONENTS_BOX)
+        return space.wrap(box)
+
+class W_Complex64Box(ComplexBox, W_ComplexFloatingBox):
+    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex64")
+    _COMPONENTS_BOX = W_Float32Box
+
+class W_Complex128Box(ComplexBox, W_ComplexFloatingBox):
+    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex128")
+    _COMPONENTS_BOX = W_Float64Box
+
+if long_double_size == 8:
+    W_FloatLongBox = W_Float64Box
+    W_ComplexLongBox = W_Complex128Box
+
+elif long_double_size in (12, 16):
+    class W_FloatLongBox(W_FloatingBox, PrimitiveBox):
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % (long_double_size * 8))
+
+    class W_ComplexLongBox(ComplexBox, W_ComplexFloatingBox):
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % (long_double_size * 16))
+        _COMPONENTS_BOX = W_FloatLongBox
+
 class W_FlexibleBox(W_GenericBox):
-    _attrs_ = ['ofs', 'dtype', 'arr']
-    _immutable_fields_ = ['ofs']
+    _attrs_ = ['arr', 'ofs', 'dtype']
+    _immutable_fields_ = ['arr', 'ofs', 'dtype']
+
     def __init__(self, arr, ofs, dtype):
         self.arr = arr # we have to keep array alive
         self.ofs = ofs
@@ -334,11 +372,6 @@
     def get_dtype(self, space):
         return self.arr.dtype
 
- at unwrap_spec(self=W_GenericBox)
-def descr_index(space, self):
-    return space.index(self.item(space))
-
-
 class W_VoidBox(W_FlexibleBox):
     def descr_getitem(self, space, w_item):
         from pypy.module.micronumpy.types import VoidType
@@ -388,7 +421,6 @@
         # XXX assert dtype is str type
         return self
 
-
 class W_StringBox(W_CharacterBox):
     def descr__new__string_box(space, w_subtype, w_arg):
         from pypy.module.micronumpy.interp_dtype import new_string_dtype
@@ -398,7 +430,6 @@
             arr.storage[i] = arg[i]
         return W_StringBox(arr, 0, arr.dtype)
 
-
 class W_UnicodeBox(W_CharacterBox):
     def descr__new__unicode_box(space, w_subtype, w_arg):
         raise OperationError(space.w_NotImplementedError, space.wrap("Unicode is not supported yet"))
@@ -413,59 +444,6 @@
         #    arr.storage[i] = arg[i]
         return W_UnicodeBox(arr, 0, arr.dtype)
 
-
-class W_ComplexFloatingBox(W_InexactBox):
-    _attrs_ = ()
-    def descr_get_real(self, space):
-        dtype = self._COMPONENTS_BOX._get_dtype(space)
-        box = self.convert_real_to(dtype)
-        assert isinstance(box, self._COMPONENTS_BOX)
-        return space.wrap(box)
-
-    def descr_get_imag(self, space):
-        dtype = self._COMPONENTS_BOX._get_dtype(space)
-        box = self.convert_imag_to(dtype)
-        assert isinstance(box, self._COMPONENTS_BOX)
-        return space.wrap(box)
-
-
-class W_Complex64Box(ComplexBox, W_ComplexFloatingBox):
-    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex64")
-    _COMPONENTS_BOX = W_Float32Box
-
-
-class W_Complex128Box(ComplexBox, W_ComplexFloatingBox):
-    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex128")
-    _COMPONENTS_BOX = W_Float64Box
-
-if ENABLED_LONG_DOUBLE and long_double_size == 12:
-    class W_Float96Box(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float96")
-
-    W_LongDoubleBox = W_Float96Box
-
-    class W_Complex192Box(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex192")
-        _COMPONENTS_BOX = W_Float96Box
-
-    W_CLongDoubleBox = W_Complex192Box
-
-elif ENABLED_LONG_DOUBLE and long_double_size == 16:
-    class W_Float128Box(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float128")
-    W_LongDoubleBox = W_Float128Box
-
-    class W_Complex256Box(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex256")
-        _COMPONENTS_BOX = W_Float128Box
-
-    W_CLongDoubleBox = W_Complex256Box
-
-elif ENABLED_LONG_DOUBLE:
-    W_LongDoubleBox = W_Float64Box
-    W_CLongDoubleBox = W_Complex64Box
-
-
 W_GenericBox.typedef = TypeDef("generic",
     __module__ = "numpypy",
 
@@ -535,7 +513,7 @@
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_BoolBox.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_BoolBox.descr_index),
     __reduce__ = interp2app(W_BoolBox.descr_reduce),
 )
 
@@ -558,49 +536,49 @@
 W_Int8Box.typedef = TypeDef("int8", W_SignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int8Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int8Box.descr_index),
     __reduce__ = interp2app(W_Int8Box.descr_reduce),
 )
 
 W_UInt8Box.typedef = TypeDef("uint8", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt8Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt8Box.descr_index),
     __reduce__ = interp2app(W_UInt8Box.descr_reduce),
 )
 
 W_Int16Box.typedef = TypeDef("int16", W_SignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int16Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int16Box.descr_index),
     __reduce__ = interp2app(W_Int16Box.descr_reduce),
 )
 
 W_UInt16Box.typedef = TypeDef("uint16", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt16Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt16Box.descr_index),
     __reduce__ = interp2app(W_UInt16Box.descr_reduce),
 )
 
 W_Int32Box.typedef = TypeDef("int32", (W_SignedIntegerBox.typedef,) + MIXIN_32,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int32Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int32Box.descr_index),
     __reduce__ = interp2app(W_Int32Box.descr_reduce),
 )
 
 W_UInt32Box.typedef = TypeDef("uint32", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt32Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt32Box.descr_index),
     __reduce__ = interp2app(W_UInt32Box.descr_reduce),
 )
 
 W_Int64Box.typedef = TypeDef("int64", (W_SignedIntegerBox.typedef,) + MIXIN_64,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int64Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int64Box.descr_index),
     __reduce__ = interp2app(W_Int64Box.descr_reduce),
 )
 
@@ -614,7 +592,7 @@
 W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt64Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt64Box.descr_index),
     __reduce__ = interp2app(W_UInt64Box.descr_reduce),
 )
 
@@ -628,53 +606,53 @@
 
 W_Float16Box.typedef = TypeDef("float16", W_FloatingBox.typedef,
     __module__ = "numpypy",
-
     __new__ = interp2app(W_Float16Box.descr__new__.im_func),
     __reduce__ = interp2app(W_Float16Box.descr_reduce),
 )
 
 W_Float32Box.typedef = TypeDef("float32", W_FloatingBox.typedef,
     __module__ = "numpypy",
-
     __new__ = interp2app(W_Float32Box.descr__new__.im_func),
     __reduce__ = interp2app(W_Float32Box.descr_reduce),
 )
 
 W_Float64Box.typedef = TypeDef("float64", (W_FloatingBox.typedef, float_typedef),
     __module__ = "numpypy",
-
     __new__ = interp2app(W_Float64Box.descr__new__.im_func),
     __reduce__ = interp2app(W_Float64Box.descr_reduce),
 )
 
-if ENABLED_LONG_DOUBLE and long_double_size == 12:
-    W_Float96Box.typedef = TypeDef("float96", (W_FloatingBox.typedef),
+W_ComplexFloatingBox.typedef = TypeDef("complexfloating", W_InexactBox.typedef,
+    __module__ = "numpypy",
+)
+
+W_Complex64Box.typedef = TypeDef("complex64", (W_ComplexFloatingBox.typedef),
+    __module__ = "numpypy",
+    __new__ = interp2app(W_Complex64Box.descr__new__.im_func),
+    __reduce__ = interp2app(W_Complex64Box.descr_reduce),
+    real = GetSetProperty(W_ComplexFloatingBox .descr_get_real),
+    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
+)
+
+W_Complex128Box.typedef = TypeDef("complex128", (W_ComplexFloatingBox.typedef, complex_typedef),
+    __module__ = "numpypy",
+    __new__ = interp2app(W_Complex128Box.descr__new__.im_func),
+    __reduce__ = interp2app(W_Complex128Box.descr_reduce),
+    real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
+    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
+)
+
+if long_double_size in (12, 16):
+    W_FloatLongBox.typedef = TypeDef("float%d" % (long_double_size * 8), (W_FloatingBox.typedef),
         __module__ = "numpypy",
-        __reduce__ = interp2app(W_Float96Box.descr_reduce),
-
-        __new__ = interp2app(W_Float96Box.descr__new__.im_func),
+        __new__ = interp2app(W_FloatLongBox.descr__new__.im_func),
+        __reduce__ = interp2app(W_FloatLongBox.descr_reduce),
     )
 
-    W_Complex192Box.typedef = TypeDef("complex192", (W_ComplexFloatingBox.typedef, complex_typedef),
+    W_ComplexLongBox.typedef = TypeDef("complex%d" % (long_double_size * 16), (W_ComplexFloatingBox.typedef, complex_typedef),
         __module__ = "numpypy",
-        __new__ = interp2app(W_Complex192Box.descr__new__.im_func),
-        __reduce__ = interp2app(W_Complex192Box.descr_reduce),
-        real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
-        imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
-    )
-
-elif ENABLED_LONG_DOUBLE and long_double_size == 16:
-    W_Float128Box.typedef = TypeDef("float128", (W_FloatingBox.typedef),
-        __module__ = "numpypy",
-
-        __new__ = interp2app(W_Float128Box.descr__new__.im_func),
-        __reduce__ = interp2app(W_Float128Box.descr_reduce),
-    )
-
-    W_Complex256Box.typedef = TypeDef("complex256", (W_ComplexFloatingBox.typedef, complex_typedef),
-        __module__ = "numpypy",
-        __new__ = interp2app(W_Complex256Box.descr__new__.im_func),
-        __reduce__ = interp2app(W_Complex256Box.descr_reduce),
+        __new__ = interp2app(W_ComplexLongBox.descr__new__.im_func),
+        __reduce__ = interp2app(W_ComplexLongBox.descr_reduce),
         real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
         imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
     )
@@ -703,24 +681,3 @@
     __module__ = "numpypy",
     __new__ = interp2app(W_UnicodeBox.descr__new__unicode_box.im_func),
 )
-
-W_ComplexFloatingBox.typedef = TypeDef("complexfloating", W_InexactBox.typedef,
-    __module__ = "numpypy",
-)
-
-
-W_Complex128Box.typedef = TypeDef("complex128", (W_ComplexFloatingBox.typedef, complex_typedef),
-    __module__ = "numpypy",
-    __new__ = interp2app(W_Complex128Box.descr__new__.im_func),
-    __reduce__ = interp2app(W_Complex128Box.descr_reduce),
-    real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
-    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
-)
-
-W_Complex64Box.typedef = TypeDef("complex64", (W_ComplexFloatingBox.typedef),
-    __module__ = "numpypy",
-    __new__ = interp2app(W_Complex64Box.descr__new__.im_func),
-    __reduce__ = interp2app(W_Complex64Box.descr_reduce),
-    real = GetSetProperty(W_ComplexFloatingBox .descr_get_real),
-    imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
-)
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -1,4 +1,3 @@
-
 import sys
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, operationerrfmt
@@ -11,6 +10,12 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib import jit
 
+if sys.byteorder == 'little':
+    byteorder_prefix = '<'
+    nonnative_byteorder_prefix = '>'
+else:
+    byteorder_prefix = '>'
+    nonnative_byteorder_prefix = '<'
 
 UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
@@ -44,12 +49,11 @@
     out = base.W_NDimArray.from_shape(space, shape, dtype)
     return out
 
-
 class W_Dtype(W_Root):
     _immutable_fields_ = ["itemtype", "num", "kind", "shape"]
 
     def __init__(self, itemtype, num, kind, name, char, w_box_type,
-                 alternate_constructors=[], aliases=[],
+                 alternate_constructors=[], aliases=[], float_type=None,
                  fields=None, fieldnames=None, native=True, shape=[], subdtype=None):
         self.itemtype = itemtype
         self.num = num
@@ -59,10 +63,10 @@
         self.w_box_type = w_box_type
         self.alternate_constructors = alternate_constructors
         self.aliases = aliases
+        self.float_type = float_type
         self.fields = fields
         self.fieldnames = fieldnames
         self.native = native
-        self.float_type = None
         self.shape = list(shape)
         self.subdtype = subdtype
         if not subdtype:
@@ -148,7 +152,11 @@
 
     def eq(self, space, w_other):
         w_other = space.call_function(space.gettypefor(W_Dtype), w_other)
-        return space.is_w(self, w_other)
+        if space.is_w(self, w_other):
+            return True
+        if isinstance(w_other, W_Dtype):
+            return space.eq_w(self.descr_reduce(space), w_other.descr_reduce(space))
+        return False
 
     def descr_eq(self, space, w_other):
         return space.wrap(self.eq(space, w_other))
@@ -223,7 +231,7 @@
         return self.kind == SIGNEDLTR
 
     def is_complex_type(self):
-        return False
+        return self.kind == COMPLEXLTR
 
     def is_float_type(self):
         return (self.kind == FLOATINGLTR or self.float_type is not None)
@@ -259,21 +267,22 @@
         builder_args = space.newtuple([space.wrap("%s%d" % (kind, elemsize)), space.wrap(0), space.wrap(1)])
 
         version = space.wrap(3)
-        order = space.wrap(byteorder_prefix if self.native else nonnative_byteorder_prefix)
         names = self.descr_get_names(space)
         values = self.descr_get_fields(space)
         if self.fields:
+            order = space.wrap('|')
             #TODO: Implement this when subarrays are implemented
             subdescr = space.w_None
-            #TODO: Change this when alignment is implemented :
             size = 0
             for key in self.fields:
                 dtype = self.fields[key][1]
                 assert isinstance(dtype, W_Dtype)
                 size += dtype.get_size()
             w_size = space.wrap(size)
+            #TODO: Change this when alignment is implemented
             alignment = space.wrap(1)
         else:
+            order = space.wrap(byteorder_prefix if self.native else nonnative_byteorder_prefix)
             subdescr = space.w_None
             w_size = space.wrap(-1)
             alignment = space.wrap(-1)
@@ -295,18 +304,6 @@
         fields = space.getitem(w_data, space.wrap(4))
         self.set_fields(space, fields)
 
-class W_ComplexDtype(W_Dtype):
-    def __init__(self, itemtype, num, kind, name, char, w_box_type,
-                 alternate_constructors=[], aliases=[],
-                 fields=None, fieldnames=None, native=True, float_type=None):
-        W_Dtype.__init__(self, itemtype, num, kind, name, char, w_box_type,
-                 alternate_constructors=alternate_constructors, aliases=aliases,
-                 fields=fields, fieldnames=fieldnames, native=native)
-        self.float_type = float_type
-
-    def is_complex_type(self):
-        return True
-
 def dtype_from_list(space, w_lst):
     lst_w = space.listview(w_lst)
     fields = {}
@@ -340,38 +337,6 @@
     raise OperationError(space.w_NotImplementedError, space.wrap(
         "dtype from dict"))
 
-def variable_dtype(space, name):
-    if name[0] in '<>=':
-        name = name[1:]
-    char = name[0]
-    if len(name) == 1:
-        size = 0
-    else:
-        try:
-            size = int(name[1:])
-        except ValueError:
-            raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
-    if char == 'S' or char == 'c':
-        itemtype = types.StringType(size)
-        basename = 'string'
-        num = 18
-        w_box_type = space.gettypefor(interp_boxes.W_StringBox)
-    elif char == 'V':
-        num = 20
-        basename = 'void'
-        itemtype = types.VoidType(size)
-        return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(size),
-                    "V", space.gettypefor(interp_boxes.W_VoidBox))
-    else:
-        assert char == 'U'
-        basename = 'unicode'
-        itemtype = types.UnicodeType(size)
-        num = 19
-        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
-    return W_Dtype(itemtype, num, char,
-                   basename + str(8 * itemtype.get_element_size()),
-                   char, w_box_type)
-
 def dtype_from_spec(space, name):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "dtype from spec"))
@@ -455,30 +420,61 @@
 )
 W_Dtype.typedef.acceptable_as_base_class = False
 
-if sys.byteorder == 'little':
-    byteorder_prefix = '<'
-    nonnative_byteorder_prefix = '>'
-else:
-    byteorder_prefix = '>'
-    nonnative_byteorder_prefix = '<'
+
+def variable_dtype(space, name):
+    if name[0] in '<>=':
+        name = name[1:]
+    char = name[0]
+    if len(name) == 1:
+        size = 0
+    else:
+        try:
+            size = int(name[1:])
+        except ValueError:
+            raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+    if char == 'c':
+        char = 'S'
+        size = 1
+    if char == 'S':
+        itemtype = types.StringType(size)
+        basename = 'string'
+        num = 18
+        w_box_type = space.gettypefor(interp_boxes.W_StringBox)
+    elif char == 'V':
+        num = 20
+        basename = 'void'
+        itemtype = types.VoidType(size)
+        return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(size),
+                    "V", space.gettypefor(interp_boxes.W_VoidBox))
+    else:
+        assert char == 'U'
+        basename = 'unicode'
+        itemtype = types.UnicodeType(size)
+        num = 19
+        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
+    return W_Dtype(itemtype, num, char,
+                   basename + str(8 * itemtype.get_element_size()),
+                   char, w_box_type)
 
 def new_string_dtype(space, size):
+    itemtype = types.StringType(size)
     return W_Dtype(
-        types.StringType(size),
+        itemtype,
         num=18,
         kind=STRINGLTR,
-        name='string',
-        char='S' + str(size),
+        name='string' + str(8 * itemtype.get_element_size()),
+        char='S',
         w_box_type = space.gettypefor(interp_boxes.W_StringBox),
     )
 
 def new_unicode_dtype(space, size):
+    itemtype = types.UnicodeType(size)
     return W_Dtype(
-        types.UnicodeType(size),
+        itemtype,
         num=19,
         kind=UNICODELTR,
-        name='unicode',
-        char='U' + str(size),
+        name='unicode' + str(8 * itemtype.get_element_size()),
+        char='U',
         w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
     )
 
@@ -542,15 +538,11 @@
             char="I",
             w_box_type=space.gettypefor(interp_boxes.W_UInt32Box),
         )
-        if LONG_BIT == 32:
-            name = "int32"
-        elif LONG_BIT == 64:
-            name = "int64"
         self.w_longdtype = W_Dtype(
             types.Long(),
             num=7,
             kind=SIGNEDLTR,
-            name=name,
+            name="int%d" % LONG_BIT,
             char="l",
             w_box_type=space.gettypefor(interp_boxes.W_LongBox),
             alternate_constructors=[space.w_int,
@@ -563,7 +555,7 @@
             types.ULong(),
             num=8,
             kind=UNSIGNEDLTR,
-            name="u" + name,
+            name="uint%d" % LONG_BIT,
             char="L",
             w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
             alternate_constructors=[ space.gettypefor(interp_boxes.W_UnsignedIntegerBox),
@@ -607,7 +599,16 @@
                                    ],
             aliases=["float", "double"],
         )
-        self.w_complex64dtype = W_ComplexDtype(
+        self.w_floatlongdtype = W_Dtype(
+            types.FloatLong(),
+            num=13,
+            kind=FLOATINGLTR,
+            name="float%d" % (interp_boxes.long_double_size * 8),
+            char="g",
+            w_box_type=space.gettypefor(interp_boxes.W_FloatLongBox),
+            aliases=["longdouble", "longfloat"],
+        )
+        self.w_complex64dtype = W_Dtype(
             types.Complex64(),
             num=14,
             kind=COMPLEXLTR,
@@ -616,7 +617,7 @@
             w_box_type = space.gettypefor(interp_boxes.W_Complex64Box),
             float_type = self.w_float32dtype,
         )
-        self.w_complex128dtype = W_ComplexDtype(
+        self.w_complex128dtype = W_Dtype(
             types.Complex128(),
             num=15,
             kind=COMPLEXLTR,
@@ -627,57 +628,16 @@
             aliases=["complex"],
             float_type = self.w_float64dtype,
         )
-        if interp_boxes.ENABLED_LONG_DOUBLE and interp_boxes.long_double_size == 12:
-            self.w_float96dtype = W_Dtype(
-                types.Float96(),
-                num=13,
-                kind=FLOATINGLTR,
-                name="float96",
-                char="g",
-                w_box_type=space.gettypefor(interp_boxes.W_Float96Box),
-                aliases=["longdouble", "longfloat"],
-            )
-            self.w_complex192dtype = W_ComplexDtype(
-                types.Complex192(),
-                num=16,
-                kind=COMPLEXLTR,
-                name="complex192",
-                char="G",
-                w_box_type = space.gettypefor(interp_boxes.W_Complex192Box),
-                alternate_constructors=[space.w_complex],
-                aliases=["clongdouble", "clongfloat"],
-                float_type = self.w_float96dtype,
-            )
-            self.w_longdouble = self.w_float96dtype
-            self.w_clongdouble = self.w_complex192dtype
-        elif interp_boxes.ENABLED_LONG_DOUBLE and interp_boxes.long_double_size == 16:
-            self.w_float128dtype = W_Dtype(
-                types.Float128(),
-                num=13,
-                kind=FLOATINGLTR,
-                name="float128",
-                char="g",
-                w_box_type=space.gettypefor(interp_boxes.W_Float128Box),
-                aliases=["longdouble", "longfloat"],
-            )
-            self.w_complex256dtype = W_ComplexDtype(
-                types.Complex256(),
-                num=16,
-                kind=COMPLEXLTR,
-                name="complex256",
-                char="G",
-                w_box_type = space.gettypefor(interp_boxes.W_Complex256Box),
-                alternate_constructors=[space.w_complex],
-                aliases=["clongdouble", "clongfloat"],
-                float_type = self.w_float128dtype,
-            )
-            self.w_longdouble = self.w_float128dtype
-            self.w_clongdouble = self.w_complex256dtype
-        elif interp_boxes.ENABLED_LONG_DOUBLE:
-            self.w_float64dtype.aliases += ["longdouble", "longfloat"]
-            self.w_complex128dtype.aliases += ["clongdouble", "clongfloat"]
-            self.w_longdouble = self.w_float64dtype
-            self.w_clongdouble = self.w_complex128dtype
+        self.w_complexlongdtype = W_Dtype(
+            types.ComplexLong(),
+            num=16,
+            kind=COMPLEXLTR,
+            name="complex%d" % (interp_boxes.long_double_size * 16),
+            char="G",
+            w_box_type = space.gettypefor(interp_boxes.W_ComplexLongBox),
+            aliases=["clongdouble", "clongfloat"],
+            float_type = self.w_floatlongdtype,
+        )
         self.w_stringdtype = W_Dtype(
             types.StringType(0),
             num=18,
@@ -750,21 +710,18 @@
             char=UINTPLTR,
             w_box_type = space.gettypefor(uintp_box),
         )
-        float_dtypes = [self.w_float16dtype,
-                self.w_float32dtype, self.w_float64dtype,
-                ]
-        complex_dtypes =  [self.w_complex64dtype, self.w_complex128dtype]
-        if interp_boxes.ENABLED_LONG_DOUBLE:
-            float_dtypes.append(self.w_longdouble)
-            complex_dtypes.append(self.w_clongdouble)
+        float_dtypes = [self.w_float16dtype, self.w_float32dtype,
+                        self.w_float64dtype, self.w_floatlongdtype]
+        complex_dtypes = [self.w_complex64dtype, self.w_complex128dtype,
+                          self.w_complexlongdtype]
         self.builtin_dtypes = [
             self.w_booldtype,
             self.w_int8dtype, self.w_uint8dtype,
             self.w_int16dtype, self.w_uint16dtype,
             self.w_longdtype, self.w_ulongdtype,
             self.w_int32dtype, self.w_uint32dtype,
-            self.w_int64dtype, self.w_uint64dtype] + \
-            float_dtypes + complex_dtypes + [
+            self.w_int64dtype, self.w_uint64dtype,
+            ] + float_dtypes + complex_dtypes + [
             self.w_stringdtype, self.w_unicodedtype, self.w_voiddtype,
             self.w_intpdtype, self.w_uintpdtype,
         ]
@@ -818,6 +775,7 @@
             'STRING': self.w_stringdtype,
             'CFLOAT': self.w_complex64dtype,
             'CDOUBLE': self.w_complex128dtype,
+            'CLONGDOUBLE': self.w_complexlongdtype,
             #'DATETIME',
             'UINT': self.w_uint32dtype,
             'INTP': self.w_intpdtype,
@@ -827,13 +785,11 @@
             #'TIMEDELTA',
             'INT': self.w_int32dtype,
             'DOUBLE': self.w_float64dtype,
+            'LONGDOUBLE': self.w_floatlongdtype,
             'USHORT': self.w_uint16dtype,
             'FLOAT': self.w_float32dtype,
             'BOOL': self.w_booldtype,
         }
-        if interp_boxes.ENABLED_LONG_DOUBLE:
-            typeinfo_full['LONGDOUBLE'] = self.w_longdouble
-            typeinfo_full['CLONGDOUBLE'] = self.w_clongdouble
 
         typeinfo_partial = {
             'Generic': interp_boxes.W_GenericBox,
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,4 +1,3 @@
-
 from pypy.interpreter.error import operationerrfmt, OperationError
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -14,7 +13,7 @@
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy import loop
 from pypy.module.micronumpy.dot import match_dot_shapes
-from pypy.module.micronumpy.interp_arrayops import repeat, choose
+from pypy.module.micronumpy.interp_arrayops import repeat, choose, put
 from pypy.module.micronumpy.arrayimpl import scalar
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rlib import jit
@@ -399,6 +398,10 @@
     def descr_repeat(self, space, repeats, w_axis=None):
         return repeat(space, self, repeats, w_axis)
 
+    def descr_set_flatiter(self, space, w_obj):
+        arr = convert_to_array(space, w_obj)
+        loop.flatiter_setitem(space, self, arr, 0, 1, self.get_size())
+
     def descr_get_flatiter(self, space):
         return space.wrap(W_FlatIterator(self))
 
@@ -417,8 +420,8 @@
                                      [0] * len(self.get_shape()))
                 assert isinstance(w_obj, interp_boxes.W_GenericBox)
                 return w_obj.item(space)
-            raise OperationError(space.w_IndexError,
-                                 space.wrap("index out of bounds"))
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("can only convert an array of size 1 to a Python scalar"))
         if space.isinstance_w(w_arg, space.w_int):
             if self.is_scalar():
                 raise OperationError(space.w_IndexError,
@@ -505,9 +508,8 @@
             loop.byteswap(self.implementation, w_res.implementation)
             return w_res
 
-    @unwrap_spec(mode=str)
-    def descr_choose(self, space, w_choices, w_out=None, mode='raise'):
-        return choose(space, self, w_choices, w_out, mode)
+    def descr_choose(self, space, w_choices, w_out=None, w_mode=None):
+        return choose(space, self, w_choices, w_out, w_mode)
 
     def descr_clip(self, space, w_min, w_max, w_out=None):
         if space.is_none(w_out):
@@ -546,6 +548,12 @@
         return interp_arrayops.diagonal(space, self.implementation, offset,
                                         axis1, axis2)
 
+    @unwrap_spec(offset=int, axis1=int, axis2=int)
+    def descr_trace(self, space, offset=0, axis1=0, axis2=1,
+                    w_dtype=None, w_out=None):
+        diag = self.descr_diagonal(space, offset, axis1, axis2)
+        return diag.descr_sum(space, w_axis=space.wrap(-1), w_dtype=w_dtype, w_out=w_out)
+
     def descr_dump(self, space, w_file):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "dump not implemented yet"))
@@ -580,10 +588,8 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "ptp (peak to peak) not implemented yet"))
 
-    @unwrap_spec(mode=str)
-    def descr_put(self, space, w_indices, w_values, mode='raise'):
-        from pypy.module.micronumpy.interp_arrayops import put
-        put(space, self, w_indices, w_values, mode)
+    def descr_put(self, space, w_indices, w_values, w_mode=None):
+        put(space, self, w_indices, w_values, w_mode)
 
     def descr_resize(self, space, w_new_shape, w_refcheck=True):
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -649,11 +655,6 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "tofile not implemented yet"))
 
-    def descr_trace(self, space, w_offset=0, w_axis1=0, w_axis2=1,
-                    w_dtype=None, w_out=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "trace not implemented yet"))
-
     def descr_view(self, space, w_dtype=None, w_type=None) :
         if not w_type and w_dtype:
             try:
@@ -841,7 +842,7 @@
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False,
                            cumultative=False):
-        def impl(self, space, w_axis=None, w_out=None, w_dtype=None):
+        def impl(self, space, w_axis=None, w_dtype=None, w_out=None):
             if space.is_none(w_out):
                 out = None
             elif not isinstance(w_out, W_NDimArray):
@@ -1130,7 +1131,8 @@
     repeat = interp2app(W_NDimArray.descr_repeat),
     swapaxes = interp2app(W_NDimArray.descr_swapaxes),
     nonzero = interp2app(W_NDimArray.descr_nonzero),
-    flat = GetSetProperty(W_NDimArray.descr_get_flatiter),

From noreply at buildbot.pypy.org  Tue Oct 22 12:45:55 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 12:45:55 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: (fijal, arigo)
Message-ID: <20131022104555.2DCF91C0149@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67503:0422d725e21c
Date: 2013-10-22 12:45 +0200
http://bitbucket.org/pypy/pypy/changeset/0422d725e21c/

Log:	(fijal, arigo)

	Support llmemory.ArrayLengthOffset to point to the two kinds of
	lengths

diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py
--- a/rpython/rtyper/lltypesystem/llmemory.py
+++ b/rpython/rtyper/lltypesystem/llmemory.py
@@ -306,8 +306,15 @@
 
 class ArrayLengthOffset(AddressOffset):
 
-    def __init__(self, TYPE):
+    def __init__(self, TYPE, attrkind="length"):
+        assert isinstance(TYPE, lltype.Array)
+        assert not TYPE._hints.get('nolength', False)
+        if attrkind == "length":
+            assert not TYPE._is_overallocated_array()
+        else:
+            assert TYPE._is_overallocated_array()
         self.TYPE = TYPE
+        self.attrkind = attrkind
 
     def __repr__(self):
         return '< ArrayLengthOffset %r >' % (self.TYPE,)
@@ -317,7 +324,8 @@
 
     def ref(self, arrayptr):
         assert array_type_match(lltype.typeOf(arrayptr).TO, self.TYPE)
-        return lltype._arraylenref._makeptr(arrayptr._obj, arrayptr._solid)
+        return lltype._arraylenref._makeptr(arrayptr._obj, arrayptr._solid,
+                                            self.attrkind)
 
 
 class GCHeaderOffset(AddressOffset):
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1192,13 +1192,7 @@
                 return
         if self._T._is_overallocated_array():
             if field_name == 'used_length':
-                if val > self._obj.getlength():
-                    raise ValueError("overallocated array size is %s, trying "
-                                     "to set used size to %s" %
-                                     (self._obj.getlength(), val))
-                for i in range(val, self._obj.used_len):
-                    self._obj.items[i] = self._T.OF._allocate('malloc')
-                self._obj.used_len = val
+                self._obj.set_used_length(val)
                 return
         raise AttributeError("%r instance has no field %r" % (self._T,
                                                               field_name))
@@ -1755,6 +1749,16 @@
         assert typeOf(value) == self._TYPE.OF
         self.items[index] = value
 
+    def set_used_length(self, val):
+        assert self._TYPE._is_overallocated_array()
+        if val > self.getlength():
+            raise ValueError("overallocated array size is %s, trying "
+                             "to set used size to %s" %
+                             (self.getlength(), val))
+        for i in range(val, self.used_len):
+            self.items[i] = self._TYPE.OF._allocate('malloc')
+        self.used_len = val
+
 assert not '__dict__' in dir(_struct)
 
 
@@ -1868,12 +1872,18 @@
     Only used internally by llmemory to implement ArrayLengthOffset.
     """
     _kind = "arraylenptr"
-    _cache = weakref.WeakKeyDictionary()  # array -> _arraylenref
+    _cache = {
+        "length": weakref.WeakKeyDictionary(),  # array -> _arraylenref
+        "allocated_length": weakref.WeakKeyDictionary(),
+        "used_length": weakref.WeakKeyDictionary(),
+        }
 
-    def __init__(self, array):
+    def __init__(self, array, attrkind):
         TYPE = FixedSizeArray(Signed, 1)
         _parentable.__init__(self, TYPE)
         self.array = array
+        self.attrkind = attrkind
+        assert attrkind in self._cache
 
     def getlength(self):
         return 1
@@ -1883,23 +1893,34 @@
 
     def getitem(self, index, uninitialized_ok=False):
         assert index == 0
-        return self.array.getlength()
+        if self.attrkind == "length":
+            return self.array.getlength()
+        elif self.attrkind == "allocated_length":
+            return len(self.array.items)
+        elif self.attrkind == "used_length":
+            return self.array.used_len
 
     def setitem(self, index, value):
         assert index == 0
-        if value != self.array.getlength():
-            if value > self.array.getlength():
-                raise Exception("can't grow an array in-place")
-            self.array.shrinklength(value)
+        if self.attrkind == "length":
+            if value != self.array.getlength():
+                if value > self.array.getlength():
+                    raise Exception("can't grow an array in-place")
+                self.array.shrinklength(value)
+        elif self.attrkind == "allocated_length":
+            raise Exception("can't set allocated_length")
+        elif self.attrkind == "used_length":
+            self.array.set_used_length(value)
 
-    def _makeptr(array, solid=False):
+    @staticmethod
+    def _makeptr(array, solid=False, attrkind="length"):
+        cache = _arraylenref._cache[attrkind]
         try:
-            lenref = _arraylenref._cache[array]
+            lenref = cache[array]
         except KeyError:
-            lenref = _arraylenref(array)
-            _arraylenref._cache[array] = lenref
+            lenref = _arraylenref(array, attrkind)
+            cache[array] = lenref
         return _ptr(Ptr(lenref._TYPE), lenref, solid)
-    _makeptr = staticmethod(_makeptr)
 
     def _getid(self):
         raise NotImplementedError('_arraylenref._getid()')
diff --git a/rpython/rtyper/lltypesystem/test/test_llmemory.py b/rpython/rtyper/lltypesystem/test/test_llmemory.py
--- a/rpython/rtyper/lltypesystem/test/test_llmemory.py
+++ b/rpython/rtyper/lltypesystem/test/test_llmemory.py
@@ -649,3 +649,16 @@
     #assert cast_int_to_adr(i) == adr -- depends on ll2ctypes details
     i = cast_adr_to_int(NULL, mode="forced")
     assert is_valid_int(i) and i == 0
+
+def test_overallocated_array():
+    A = lltype.GcArray(lltype.Signed, hints={'overallocated': True})
+    a = lltype.malloc(A, 10)
+    adr = cast_ptr_to_adr(a)
+    py.test.raises(AssertionError, ArrayLengthOffset, A)
+    length_adr = adr + ArrayLengthOffset(A, "allocated_length")
+    assert length_adr.signed[0] == 10
+    length_adr = adr + ArrayLengthOffset(A, "used_length")
+    assert length_adr.signed[0] == 0
+    #
+    length_adr.signed[0] = 2
+    assert a.used_length == 2

From noreply at buildbot.pypy.org  Tue Oct 22 12:55:31 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 12:55:31 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: (fijal, arigo)
Message-ID: <20131022105531.D2DCC1C0149@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67504:1a8508ac5763
Date: 2013-10-22 12:54 +0200
http://bitbucket.org/pypy/pypy/changeset/1a8508ac5763/

Log:	(fijal, arigo)

	The first translated test passes. The 2nd not

diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -543,6 +543,9 @@
             if isinstance(TYPE, lltype.Struct):
                 offset_to_length = llmemory.FieldOffset(TYPE, TYPE._arrayfld) + \
                                    llmemory.ArrayLengthOffset(ARRAY)
+            elif TYPE._is_overallocated_array():
+                offset_to_length = llmemory.ArrayLengthOffset(ARRAY,
+                                         attrkind="allocated_length")
             else:
                 offset_to_length = llmemory.ArrayLengthOffset(ARRAY)
             c_offset_to_length = intconst(offset_to_length)
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -508,9 +508,28 @@
             return '%s = %d;' % (self.expr(op.result),
                                  ARRAY.length)
         else:
+            assert not ARRAY._is_overallocated_array()
             return '%s = %s->length;' % (self.expr(op.result),
                                          self.expr(op.args[0]))
 
+    def OP_GETARRAYALLOCATEDLENGTH(self, op):
+        ARRAY = self.lltypemap(op.args[0]).TO
+        assert ARRAY._is_overallocated_array()
+        return '%s = %s->allocated_length;' % (self.expr(op.result),
+                                               self.expr(op.args[0]))
+
+    def OP_GETARRAYUSEDLENGTH(self, op):
+        ARRAY = self.lltypemap(op.args[0]).TO
+        assert ARRAY._is_overallocated_array()
+        return '%s = %s->used_length;' % (self.expr(op.result),
+                                          self.expr(op.args[0]))
+
+    def OP_SETARRAYUSEDLENGTH(self, op):
+        ARRAY = self.lltypemap(op.args[0]).TO
+        assert ARRAY._is_overallocated_array()
+        return 'RPySetUsedLength(%s, %s);' % (self.expr(op.args[0]),
+                                              self.expr(op.args[1]))
+
     def OP_GETARRAYITEM(self, op):
         ARRAY = self.lltypemap(op.args[0]).TO
         ptr = self.expr(op.args[0])
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -254,6 +254,8 @@
     def itemindex_access_expr(self, baseexpr, indexexpr):
         if self.ARRAY._hints.get('nolength', False):
             return 'RPyNLenItem(%s, %s)' % (baseexpr, indexexpr)
+        elif self.ARRAY._hints.get('overallocated', False):
+            return 'RPyOAItem(%s, %s)' % (baseexpr, indexexpr)
         else:
             return 'RPyItem(%s, %s)' % (baseexpr, indexexpr)
 
@@ -261,7 +263,12 @@
         yield 'struct %s {' % self.name
         for fname, typename in self.gcfields:
             yield '\t' + cdecl(typename, fname) + ';'
-        if not self.ARRAY._hints.get('nolength', False):
+        if self.ARRAY._hints.get('nolength', False):
+            pass
+        elif self.ARRAY._hints.get('overallocated', False):
+            yield '\tlong used_length;'
+            yield '\tlong allocated_length;'
+        else:
             yield '\tlong length;'
         line = '%s;' % cdecl(self.itemtypename,
                              'items[%s]' % deflength(self.varlength))
@@ -684,6 +691,8 @@
                 yield line
         if T._hints.get('nolength', False):
             length = ''
+        elif T._hints.get('overallocated', False):
+            xxxxxxxxxxx
         else:
             length = '%d, ' % len(self.obj.items)
         if T.OF is Void or len(self.obj.items) == 0:
diff --git a/rpython/translator/c/primitive.py b/rpython/translator/c/primitive.py
--- a/rpython/translator/c/primitive.py
+++ b/rpython/translator/c/primitive.py
@@ -59,8 +59,9 @@
             else:
                 return 'sizeof(%s)' % (cdecl(db.gettype(value.TYPE), ''),)
         elif isinstance(value, ArrayLengthOffset):
-            return 'offsetof(%s, length)' % (
-                cdecl(db.gettype(value.TYPE), ''))
+            return 'offsetof(%s, %s)' % (
+                cdecl(db.gettype(value.TYPE), ''),
+                value.attrkind)   # "length", "allocated_length", "used_length"
         elif isinstance(value, CompositeOffset):
             names = [name_signed(item, db) for item in value.offsets]
             return '(%s)' % (' + '.join(names),)
diff --git a/rpython/translator/c/src/support.h b/rpython/translator/c/src/support.h
--- a/rpython/translator/c/src/support.h
+++ b/rpython/translator/c/src/support.h
@@ -52,6 +52,9 @@
 #  define RPyItem(array, index)                                             \
      ((RPyCHECK((index) >= 0 && (index) < (array)->length),                 \
       (array))->items[index])
+#  define RPyOAItem(array, index)                                           \
+     ((RPyCHECK((index) >= 0 && (index) < (array)->used_length),            \
+      (array))->items[index])
 #  define RPyFxItem(ptr, index, fixedsize)                                  \
      ((RPyCHECK((ptr) && (index) >= 0 && (index) < (fixedsize)),            \
       (ptr))[index])
@@ -59,11 +62,17 @@
      ((RPyCHECK((array) && (index) >= 0), (array))->items[index])
 #  define RPyBareItem(array, index)                                         \
      ((RPyCHECK((array) && (index) >= 0), (array))[index])
+#  define RPySetUsedLength(array, nlength)                                  \
+     ((RPyCHECK((array) && (nlength) >= 0                                   \
+                        && (nlength) <= (array)->allocated_length),         \
+       (array))->used_length = (nlength))
 
 #else
 #  define RPyField(ptr, name)                ((ptr)->name)
 #  define RPyItem(array, index)              ((array)->items[index])
+#  define RPyOAItem(array, index)            ((array)->items[index])
 #  define RPyFxItem(ptr, index, fixedsize)   ((ptr)[index])
 #  define RPyNLenItem(array, index)          ((array)->items[index])
 #  define RPyBareItem(array, index)          ((array)[index])
+#  define RPySetUsedLength(array, nlength)   ((array)->used_length = (nlength))
 #endif
diff --git a/rpython/translator/c/test/test_lltyped.py b/rpython/translator/c/test/test_lltyped.py
--- a/rpython/translator/c/test/test_lltyped.py
+++ b/rpython/translator/c/test/test_lltyped.py
@@ -960,3 +960,33 @@
 
         fn = self.getcompiled(f, [int])
         assert fn(0) == 9
+
+    def test_overallocated_array(self):
+        A = GcArray(Signed, hints={'overallocated': True})
+
+        def f():
+            a = malloc(A, 10)
+            a.used_length = 5
+            a[3] = 42
+            assert a[3] == 42
+            return a.used_length + (a.allocated_length * 100)
+
+        fn = self.getcompiled(f, [])
+        assert fn() == 1005
+
+    def test_overallocated_array_prebuilt(self):
+        A = GcArray(Signed, hints={'overallocated': True})
+        a = malloc(A, 10)
+        a.used_length = 2
+        a[0] = 42
+        a[1] = 421
+
+        def f():
+            assert a.used_length == 2
+            assert a.allocated_length == 2  # reduced to its min by translation
+            assert a[0] == 42
+            assert a[1] == 421
+            return 1
+
+        fn = self.getcompiled(f, [])
+        assert fn() == 1

From noreply at buildbot.pypy.org  Tue Oct 22 13:03:31 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 13:03:31 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: Fix the 2nd
	test too.
Message-ID: <20131022110331.3D1921C0149@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67505:299d89ba4157
Date: 2013-10-22 13:02 +0200
http://bitbucket.org/pypy/pypy/changeset/299d89ba4157/

Log:	Fix the 2nd test too.

diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -675,6 +675,8 @@
         return 'array'
 
     def enum_dependencies(self):
+        if self.getTYPE()._hints.get('overallocated', False):
+            return self.obj.items[:self.obj.used_len]
         return self.obj.items
 
     def getvarlength(self):
@@ -689,14 +691,18 @@
                                                '%sgcheader' % (decoration,))
             for line in lines:
                 yield line
+        length = len(self.obj.items)
         if T._hints.get('nolength', False):
-            length = ''
+            slength = ''
         elif T._hints.get('overallocated', False):
-            xxxxxxxxxxx
+            # during translation, we reduce the over-allocated arrays
+            # to their used_len
+            length = self.obj.used_len
+            slength = '%d, %d, ' % (length, length)
         else:
-            length = '%d, ' % len(self.obj.items)
+            slength = '%d, ' % length
         if T.OF is Void or len(self.obj.items) == 0:
-            yield '\t%s' % length.rstrip(', ')
+            yield '\t%s' % slength.rstrip(', ')
             yield '}'
         elif T.OF == Char:
             if len(self.obj.items) and self.obj.items[0] is None:
@@ -707,13 +713,13 @@
             if array_constant.startswith('{') and barebonearray(T):
                 assert array_constant.endswith('}')
                 array_constant = array_constant[1:-1].strip()
-            yield '\t%s%s' % (length, array_constant)
+            yield '\t%s%s' % (slength, array_constant)
             yield '}'
         else:
             barebone = barebonearray(T)
             if not barebone:
-                yield '\t%s{' % length
-            for j in range(len(self.obj.items)):
+                yield '\t%s{' % slength
+            for j in range(length):
                 value = self.obj.items[j]
                 basename = self.name
                 if basename.endswith('.b'):

From noreply at buildbot.pypy.org  Tue Oct 22 13:15:43 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 13:15:43 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: Detail
Message-ID: <20131022111543.0D8891C0144@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67506:a9c7ed165db9
Date: 2013-10-22 13:14 +0200
http://bitbucket.org/pypy/pypy/changeset/a9c7ed165db9/

Log:	Detail

diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -701,7 +701,7 @@
             slength = '%d, %d, ' % (length, length)
         else:
             slength = '%d, ' % length
-        if T.OF is Void or len(self.obj.items) == 0:
+        if T.OF is Void or length == 0:
             yield '\t%s' % slength.rstrip(', ')
             yield '}'
         elif T.OF == Char:

From noreply at buildbot.pypy.org  Tue Oct 22 13:26:26 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 22 Oct 2013 13:26:26 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: fix the
	array_type_match
Message-ID: <20131022112626.E26C11C0144@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: array-overallocation-in-nursery
Changeset: r67507:e3e1ca30e322
Date: 2013-10-22 13:25 +0200
http://bitbucket.org/pypy/pypy/changeset/e3e1ca30e322/

Log:	fix the array_type_match

diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py
--- a/rpython/rtyper/lltypesystem/llmemory.py
+++ b/rpython/rtyper/lltypesystem/llmemory.py
@@ -593,7 +593,8 @@
     return A1 == A2 or (A2 == GCARRAY_OF_PTR and
                         isinstance(A1, lltype.GcArray) and
                         isinstance(A1.OF, lltype.Ptr) and
-                        not A1._hints.get('nolength'))
+                        not A1._hints.get('nolength') and
+        not A1._hints.get('overallocated'))
 def array_item_type_match(T1, T2):
     return T1 == T2 or (T2 == GCREF and isinstance(T1, lltype.Ptr))
 
@@ -911,7 +912,7 @@
         assert isinstance(s_from, SomeAddress)
         assert isinstance(s_to, SomeAddress)
         assert isinstance(s_size, SomeInteger)
-    
+
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
         v_list = hop.inputargs(Address, Address, lltype.Signed)

From noreply at buildbot.pypy.org  Tue Oct 22 13:38:30 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 22 Oct 2013 13:38:30 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: (fijal,
	arigo) in-progress
Message-ID: <20131022113830.466181D2318@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: array-overallocation-in-nursery
Changeset: r67508:cc86c3b3bf85
Date: 2013-10-22 13:37 +0200
http://bitbucket.org/pypy/pypy/changeset/cc86c3b3bf85/

Log:	(fijal, arigo) in-progress

diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py
--- a/rpython/memory/gctypelayout.py
+++ b/rpython/memory/gctypelayout.py
@@ -38,11 +38,12 @@
         hints={'immutable': True},
         )
     VARSIZE_TYPE_INFO = lltype.Struct("varsize_type_info",
-        ("header",         TYPE_INFO),
-        ("varitemsize",    lltype.Signed),
-        ("ofstovar",       lltype.Signed),
-        ("ofstolength",    lltype.Signed),
-        ("varofstoptrs",   lltype.Ptr(OFFSETS_TO_GC_PTR)),
+        ("header",          TYPE_INFO),
+        ("varitemsize",     lltype.Signed),
+        ("ofstovar",        lltype.Signed),
+        ("ofstolength",     lltype.Signed), # either length or allocated_length
+        ("ofstousedlength", lltype.Signed),
+        ("varofstoptrs",    lltype.Ptr(OFFSETS_TO_GC_PTR)),
         hints={'immutable': True},
         )
     TYPE_INFO_PTR = lltype.Ptr(TYPE_INFO)
@@ -240,10 +241,13 @@
         else:
             assert isinstance(TYPE, lltype.GcArray)
             ARRAY = TYPE
-            if (isinstance(ARRAY.OF, lltype.Ptr)
-                and ARRAY.OF.TO._gckind == 'gc'):
-                infobits |= T_IS_GCARRAY_OF_GCPTR
-            varinfo.ofstolength = llmemory.ArrayLengthOffset(ARRAY)
+            if ARRAY._is_overallocated_array():
+                if (isinstance(ARRAY.OF, lltype.Ptr)
+                    and ARRAY.OF.TO._gckind == 'gc'):
+                    infobits |= T_IS_GCARRAY_OF_GCPTR
+                varinfo.ofstolength = llmemory.ArrayLengthOffset(ARRAY)
+            else:
+                ...
             varinfo.ofstovar = llmemory.itemoffsetof(TYPE, 0)
         assert isinstance(ARRAY, lltype.Array)
         if ARRAY.OF != lltype.Void:
diff --git a/rpython/memory/test/gc_test_base.py b/rpython/memory/test/gc_test_base.py
--- a/rpython/memory/test/gc_test_base.py
+++ b/rpython/memory/test/gc_test_base.py
@@ -29,6 +29,7 @@
     GC_CAN_SHRINK_ARRAY = False
     GC_CAN_SHRINK_BIG_ARRAY = False
     BUT_HOW_BIG_IS_A_BIG_STRING = 3*WORD
+    SUPPORTS_OVERALLOCATED_ARRAYS = False
 
     def setup_class(cls):
         cls._saved_logstate = py.log._getstate()
@@ -86,7 +87,7 @@
         for i in range(1, 15):
             res = self.interpret(append_to_list, [i, i - 1])
             assert res == i - 1 # crashes if constants are not considered roots
-            
+
     def test_string_concatenation(self):
         #curr = simulator.current_size
         def concat(j):
@@ -783,6 +784,29 @@
             assert rgc.get_gcflag_extra(a2) == False
         self.interpret(fn, [])
 
+    def test_overallocated_array(self):
+        S = lltype.GcStruct('S')
+        A = lltype.GcArray(lltype.Ptr(S), hints={'overallocated': True})
+        if self.SUPPORTS_OVERALLOCATED_ARRAYS:
+            EXPECTED_LENGTH = 2
+        else:
+            EXPECTED_LENGTH = 10
+
+        def fn():
+            a = lltype.malloc(A, 10)
+            a.used_length = 2
+            s1 = lltype.malloc(S)
+            s2 = lltype.malloc(S)
+            a[0] = s1
+            a[1] = s2
+            rgc.collect()
+            assert a[0] == s1
+            assert a[1] == s2
+            assert a.used_length == 2
+            assert a.allocated_length == EXPECTED_LENGTH
+
+        self.interpret(fn, [])
+
 from rpython.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):
diff --git a/rpython/memory/test/test_gctypelayout.py b/rpython/memory/test/test_gctypelayout.py
--- a/rpython/memory/test/test_gctypelayout.py
+++ b/rpython/memory/test/test_gctypelayout.py
@@ -23,7 +23,7 @@
 GC_A = lltype.GcArray(S)
 
 S2 = lltype.Struct('SPTRS',
-                   *[(getname(TYPE), lltype.Ptr(TYPE)) for TYPE in (GC_S, GC_A)])  
+                   *[(getname(TYPE), lltype.Ptr(TYPE)) for TYPE in (GC_S, GC_A)])
 GC_S2 = lltype.GcStruct('GC_S2', ('S2', S2))
 
 A2 = lltype.Array(S2)
@@ -120,3 +120,11 @@
     adr = llmemory.cast_ptr_to_adr(s3)
     lst = list(gc_pointers_inside(s3._obj, adr, mutable_only=True))
     assert lst == [adr + llmemory.offsetof(S3, 'y')]
+
+def test_overallocated_array():
+    S = lltype.GcStruct('S')
+    A = lltype.GcArray(lltype.Ptr(S), hints={'overallocated': True})
+    
+    layoutbuilder = TypeLayoutBuilder(FakeGC)
+    tid = layoutbuilder.get_type_id(A)
+    xxx
diff --git a/rpython/memory/test/test_incminimark_gc.py b/rpython/memory/test/test_incminimark_gc.py
--- a/rpython/memory/test/test_incminimark_gc.py
+++ b/rpython/memory/test/test_incminimark_gc.py
@@ -4,3 +4,6 @@
 
 class TestIncrementalMiniMarkGC(test_minimark_gc.TestMiniMarkGC):
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
+
+    SUPPORTS_OVERALLOCATED_ARRAYS = True
+

From noreply at buildbot.pypy.org  Tue Oct 22 13:47:35 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 13:47:35 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: (fijal,
 arigo) Finish and pass the test for gctypelayout.
Message-ID: <20131022114735.4DD551D2377@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67509:5148aded3637
Date: 2013-10-22 13:46 +0200
http://bitbucket.org/pypy/pypy/changeset/5148aded3637/

Log:	(fijal, arigo) Finish and pass the test for gctypelayout.

diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py
--- a/rpython/memory/gctypelayout.py
+++ b/rpython/memory/gctypelayout.py
@@ -113,6 +113,9 @@
     def q_varsize_offset_to_length(self, typeid):
         return self.get_varsize(typeid).ofstolength
 
+    def q_varsize_offset_to_used_length(self, typeid):
+        return self.get_varsize(typeid).ofstousedlength
+
     def q_varsize_offsets_to_gcpointers_in_var_part(self, typeid):
         return self.get_varsize(typeid).varofstoptrs
 
@@ -241,13 +244,15 @@
         else:
             assert isinstance(TYPE, lltype.GcArray)
             ARRAY = TYPE
-            if ARRAY._is_overallocated_array():
+            if not ARRAY._is_overallocated_array():
                 if (isinstance(ARRAY.OF, lltype.Ptr)
                     and ARRAY.OF.TO._gckind == 'gc'):
                     infobits |= T_IS_GCARRAY_OF_GCPTR
                 varinfo.ofstolength = llmemory.ArrayLengthOffset(ARRAY)
             else:
-                ...
+                ALO = llmemory.ArrayLengthOffset
+                varinfo.ofstolength = ALO(ARRAY, attrkind="allocated_length")
+                varinfo.ofstousedlength = ALO(ARRAY, attrkind="used_length")
             varinfo.ofstovar = llmemory.itemoffsetof(TYPE, 0)
         assert isinstance(ARRAY, lltype.Array)
         if ARRAY.OF != lltype.Void:
diff --git a/rpython/memory/test/test_gctypelayout.py b/rpython/memory/test/test_gctypelayout.py
--- a/rpython/memory/test/test_gctypelayout.py
+++ b/rpython/memory/test/test_gctypelayout.py
@@ -127,4 +127,12 @@
     
     layoutbuilder = TypeLayoutBuilder(FakeGC)
     tid = layoutbuilder.get_type_id(A)
-    xxx
+    gcdata = GCData(layoutbuilder.type_info_group)
+    assert gcdata.q_is_varsize(tid)
+    assert not gcdata.q_is_gcarrayofgcptr(tid)   # XXX for now
+    ofs = gcdata.q_varsize_offset_to_length(tid)
+    assert isinstance(ofs, llmemory.ArrayLengthOffset)
+    assert ofs.attrkind == "allocated_length"
+    ofs = gcdata.q_varsize_offset_to_used_length(tid)
+    assert isinstance(ofs, llmemory.ArrayLengthOffset)
+    assert ofs.attrkind == "used_length"

From noreply at buildbot.pypy.org  Tue Oct 22 13:57:00 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 13:57:00 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: For GCs that
 never shrink overallocated array, test_overallocated_array
Message-ID: <20131022115700.B40451D2377@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67510:4eb04f9f277a
Date: 2013-10-22 13:55 +0200
http://bitbucket.org/pypy/pypy/changeset/4eb04f9f277a/

Log:	For GCs that never shrink overallocated array,
	test_overallocated_array passes.

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -68,6 +68,7 @@
                             fixed_size, varsize_item_sizes,
                             varsize_offset_to_variable_part,
                             varsize_offset_to_length,
+                            varsize_offset_to_used_length,
                             varsize_offsets_to_gcpointers_in_var_part,
                             weakpointer_offset,
                             member_index,
@@ -86,6 +87,7 @@
         self.varsize_item_sizes = varsize_item_sizes
         self.varsize_offset_to_variable_part = varsize_offset_to_variable_part
         self.varsize_offset_to_length = varsize_offset_to_length
+        self.varsize_offset_to_used_length = varsize_offset_to_used_length
         self.varsize_offsets_to_gcpointers_in_var_part = varsize_offsets_to_gcpointers_in_var_part
         self.weakpointer_offset = weakpointer_offset
         self.member_index = member_index
@@ -217,7 +219,8 @@
         typeid = self.get_type_id(obj)
         if self.has_gcptr_in_varsize(typeid):
             item = obj + self.varsize_offset_to_variable_part(typeid)
-            length = (obj + self.varsize_offset_to_length(typeid)).signed[0]
+            length_adr = (obj + self.varsize_offset_to_used_length(typeid))
+            length = length_adr.signed[0]
             offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid)
             itemlength = self.varsize_item_sizes(typeid)
             while length > 0:
diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py
--- a/rpython/memory/gctypelayout.py
+++ b/rpython/memory/gctypelayout.py
@@ -164,6 +164,7 @@
             self.q_varsize_item_sizes,
             self.q_varsize_offset_to_variable_part,
             self.q_varsize_offset_to_length,
+            self.q_varsize_offset_to_used_length,
             self.q_varsize_offsets_to_gcpointers_in_var_part,
             self.q_weakpointer_offset,
             self.q_member_index,
@@ -248,11 +249,12 @@
                 if (isinstance(ARRAY.OF, lltype.Ptr)
                     and ARRAY.OF.TO._gckind == 'gc'):
                     infobits |= T_IS_GCARRAY_OF_GCPTR
-                varinfo.ofstolength = llmemory.ArrayLengthOffset(ARRAY)
+                attrkinds = "length", "length"
             else:
-                ALO = llmemory.ArrayLengthOffset
-                varinfo.ofstolength = ALO(ARRAY, attrkind="allocated_length")
-                varinfo.ofstousedlength = ALO(ARRAY, attrkind="used_length")
+                attrkinds = "allocated_length", "used_length"
+            ALO = llmemory.ArrayLengthOffset
+            varinfo.ofstolength = ALO(ARRAY, attrkind=attrkinds[0])
+            varinfo.ofstousedlength = ALO(ARRAY, attrkind=attrkinds[1])
             varinfo.ofstovar = llmemory.itemoffsetof(TYPE, 0)
         assert isinstance(ARRAY, lltype.Array)
         if ARRAY.OF != lltype.Void:
diff --git a/rpython/memory/test/gc_test_base.py b/rpython/memory/test/gc_test_base.py
--- a/rpython/memory/test/gc_test_base.py
+++ b/rpython/memory/test/gc_test_base.py
@@ -785,7 +785,7 @@
         self.interpret(fn, [])
 
     def test_overallocated_array(self):
-        S = lltype.GcStruct('S')
+        S = lltype.GcStruct('S', ('n', lltype.Signed))
         A = lltype.GcArray(lltype.Ptr(S), hints={'overallocated': True})
         if self.SUPPORTS_OVERALLOCATED_ARRAYS:
             EXPECTED_LENGTH = 2
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1908,7 +1908,8 @@
                     raise Exception("can't grow an array in-place")
                 self.array.shrinklength(value)
         elif self.attrkind == "allocated_length":
-            raise Exception("can't set allocated_length")
+            if value != len(self.array.items):
+                raise Exception("can't set allocated_length")
         elif self.attrkind == "used_length":
             self.array.set_used_length(value)
 

From noreply at buildbot.pypy.org  Tue Oct 22 14:47:33 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 14:47:33 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: (fijal,
	arigo) in-progress
Message-ID: <20131022124733.B8CF11C0403@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67511:9c69606d0734
Date: 2013-10-22 14:46 +0200
http://bitbucket.org/pypy/pypy/changeset/9c69606d0734/

Log:	(fijal, arigo) in-progress

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -131,6 +131,28 @@
     def get_size_incl_hash(self, obj):
         return self.get_size(obj)
 
+    def _shrink_and_get_size_for_typeid(self, obj, typeid):
+        # Get the size of the object; but for overallocated arrays, we
+        # first fix the allocated length in-place to reduce it to the
+        # used length.
+        size = self.fixed_size(typeid)
+        if self.is_varsize(typeid):
+            ofslen = self.varsize_offset_to_length(typeid)
+            ofsusedlen = self.varsize_offset_to_used_length(typeid)
+            lenaddr = obj + ofsusedlen
+            length = lenaddr.signed[0]
+            if not (ofsusedlen == ofslen):
+                (obj + ofslen).signed[0] = length
+            size += length * self.varsize_item_sizes(typeid)
+            size = llarena.round_up_for_allocation(size)
+            # XXX maybe we should parametrize round_up_for_allocation()
+            # per GC; if we do, we also need to fix the call in
+            # gctypelayout.encode_type_shape()
+        return size
+
+    def shrink_and_get_size(self, obj):
+        return self._shrink_and_get_size_for_typeid(obj, self.get_type_id(obj))
+
     def malloc(self, typeid, length=0, zero=False):
         """For testing.  The interface used by the gctransformer is
         the four malloc_[fixed,var]size[_clear]() functions.
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1618,7 +1618,7 @@
             # tid == -42, containing all flags), and it doesn't have the
             # HAS_SHADOW flag either.  We must move it out of the nursery,
             # into a new nonmovable location.
-            totalsize = size_gc_header + self.get_size(obj)
+            totalsize = size_gc_header + self.shrink_and_get_size(obj)
             self.nursery_surviving_size += raw_malloc_usage(totalsize)
             newhdr = self._malloc_out_of_nursery(totalsize)
             #
diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py
--- a/rpython/memory/gctypelayout.py
+++ b/rpython/memory/gctypelayout.py
@@ -241,6 +241,7 @@
             ARRAY = TYPE._flds[TYPE._arrayfld]
             ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld)
             varinfo.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY)
+            varinfo.ofstousedlength = varinfo.ofstolength
             varinfo.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0)
         else:
             assert isinstance(TYPE, lltype.GcArray)
@@ -249,12 +250,12 @@
                 if (isinstance(ARRAY.OF, lltype.Ptr)
                     and ARRAY.OF.TO._gckind == 'gc'):
                     infobits |= T_IS_GCARRAY_OF_GCPTR
-                attrkinds = "length", "length"
+                varinfo.ofstolength = llmemory.ArrayLengthOffset(ARRAY)
+                varinfo.ofstousedlength = varinfo.ofstolength
             else:
-                attrkinds = "allocated_length", "used_length"
-            ALO = llmemory.ArrayLengthOffset
-            varinfo.ofstolength = ALO(ARRAY, attrkind=attrkinds[0])
-            varinfo.ofstousedlength = ALO(ARRAY, attrkind=attrkinds[1])
+                ALO = llmemory.ArrayLengthOffset
+                varinfo.ofstolength = ALO(ARRAY, attrkind="allocated_length")
+                varinfo.ofstousedlength = ALO(ARRAY, attrkind="used_length")
             varinfo.ofstovar = llmemory.itemoffsetof(TYPE, 0)
         assert isinstance(ARRAY, lltype.Array)
         if ARRAY.OF != lltype.Void:
diff --git a/rpython/memory/test/gc_test_base.py b/rpython/memory/test/gc_test_base.py
--- a/rpython/memory/test/gc_test_base.py
+++ b/rpython/memory/test/gc_test_base.py
@@ -790,10 +790,10 @@
         if self.SUPPORTS_OVERALLOCATED_ARRAYS:
             EXPECTED_LENGTH = 2
         else:
-            EXPECTED_LENGTH = 10
+            EXPECTED_LENGTH = 3
 
         def fn():
-            a = lltype.malloc(A, 10)
+            a = lltype.malloc(A, 3)
             a.used_length = 2
             s1 = lltype.malloc(S)
             s2 = lltype.malloc(S)
@@ -806,7 +806,6 @@
             assert a.allocated_length == EXPECTED_LENGTH
 
         self.interpret(fn, [])
-
 from rpython.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):
diff --git a/rpython/memory/test/test_gctypelayout.py b/rpython/memory/test/test_gctypelayout.py
--- a/rpython/memory/test/test_gctypelayout.py
+++ b/rpython/memory/test/test_gctypelayout.py
@@ -136,3 +136,4 @@
     ofs = gcdata.q_varsize_offset_to_used_length(tid)
     assert isinstance(ofs, llmemory.ArrayLengthOffset)
     assert ofs.attrkind == "used_length"
+    assert gcdata.q_is_overallocated_array(tid)
diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py
--- a/rpython/rtyper/lltypesystem/llarena.py
+++ b/rpython/rtyper/lltypesystem/llarena.py
@@ -35,6 +35,8 @@
         return '<Arena #%d [%d bytes]>' % (self._arena_index, self.nbytes)
 
     def reset(self, zero, start=0, size=None):
+        from rpython.rtyper.lltypesystem import lltype
+        WORD = llmemory.raw_malloc_usage(llmemory.sizeof(lltype.Signed))
         self.check()
         if size is None:
             stop = self.nbytes
@@ -43,11 +45,19 @@
         assert 0 <= start <= stop <= self.nbytes
         for offset, ptr in self.objectptrs.items():
             size = self.objectsizes[offset]
+            clearme = False
             if offset < start:   # object is before the cleared area
                 assert offset + size <= start, "object overlaps cleared area"
             elif offset + size > stop:  # object is after the cleared area
-                assert offset >= stop, "object overlaps cleared area"
+                if offset in (start, start+WORD) and not zero:
+                    # accept non-zeroing reset() with a length that is
+                    # shorter than the actual length, for the gc
+                    clearme = True
+                else:
+                    assert offset >= stop, "object overlaps cleared area"
             else:
+                clearme = True
+            if clearme:
                 obj = ptr._obj
                 _dictdel(Arena.object_arena_location, obj)
                 del self.objectptrs[offset]
diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py
--- a/rpython/rtyper/lltypesystem/llmemory.py
+++ b/rpython/rtyper/lltypesystem/llmemory.py
@@ -300,8 +300,10 @@
         return cast_ptr_to_adr(p)
 
     def raw_memcopy(self, srcadr, dstadr):
-        # should really copy the length field, but we can't
-        pass
+        # should really copy the length field, but we can't.  But still
+        # we need it for the used_length field of overallocated arrays...
+        if self.TYPE._is_overallocated_array():
+            dstadr.ptr.used_length = srcadr.ptr.used_length
 
 
 class ArrayLengthOffset(AddressOffset):
@@ -316,6 +318,16 @@
         self.TYPE = TYPE
         self.attrkind = attrkind
 
+    # special-casing: only for one check in the GC
+    def __eq__(self, other):
+        if self is other:
+            return True
+        if not isinstance(other, ArrayLengthOffset):
+            return False
+        if self.attrkind != other.attrkind:
+            return False
+        raise NotImplementedError
+
     def __repr__(self):
         return '< ArrayLengthOffset %r >' % (self.TYPE,)
 
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -1909,7 +1909,9 @@
                 self.array.shrinklength(value)
         elif self.attrkind == "allocated_length":
             if value != len(self.array.items):
-                raise Exception("can't set allocated_length")
+                if value > len(self.array.items):
+                    raise Exception("can't grow allocated_length in-place")
+                self.array.shrinklength(value)
         elif self.attrkind == "used_length":
             self.array.set_used_length(value)
 

From noreply at buildbot.pypy.org  Tue Oct 22 14:50:57 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Oct 2013 14:50:57 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: start implementing repeated write and
 read barriers in stmrewrite and gc (WIP)
Message-ID: <20131022125057.7254A1C0403@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67512:bea8518a7ce2
Date: 2013-10-22 12:19 +0200
http://bitbucket.org/pypy/pypy/changeset/bea8518a7ce2/

Log:	start implementing repeated write and read barriers in stmrewrite
	and gc (WIP)

diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -428,10 +428,13 @@
 
 class STMReadBarrierDescr(STMBarrierDescr):
     def __init__(self, gc_ll_descr, stmcat):
-        assert stmcat == 'P2R'
-        STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
-                                 'stm_DirectReadBarrier') 
-        # XXX: implement fastpath then change to stm_DirectReadBarrier
+        assert stmcat in ['A2R', 'Q2R']
+        if stmcat == 'A2R':
+            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
+                                     'stm_DirectReadBarrier')
+        else:
+            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
+                                     'stm_RepeatReadBarrier')
 
     @specialize.arg(2)
     def _do_barrier(self, gcref_struct, returns_modified_object):
@@ -440,20 +443,27 @@
         objadr = llmemory.cast_ptr_to_adr(gcref_struct)
         objhdr = rffi.cast(StmGC.GCHDRP, gcref_struct)
 
-        # if h_revision == privat_rev of transaction
-        priv_rev = self.llop1.stm_get_adr_of_private_rev_num(rffi.SIGNEDP)
-        if objhdr.h_revision == priv_rev[0]:
-            return gcref_struct
+        if self.stmcat == 'A2R':
+            # if h_revision == privat_rev of transaction
+            priv_rev = self.llop1.stm_get_adr_of_private_rev_num(rffi.SIGNEDP)
+            if objhdr.h_revision == priv_rev[0]:
+                return gcref_struct
 
-        # readcache[obj] == obj
-        read_cache = self.llop1.stm_get_adr_of_read_barrier_cache(rffi.SIGNEDP)
-        objint = llmemory.cast_adr_to_int(objadr)
-        assert WORD == 8, "check for 32bit compatibility"
-        index = (objint & StmGC.FX_MASK) / WORD
-        CP = lltype.Ptr(rffi.CArray(lltype.Signed))
-        rcp = rffi.cast(CP, read_cache[0])
-        if rcp[index] == objint:
-            return gcref_struct
+            # readcache[obj] == obj
+            read_cache = self.llop1.stm_get_adr_of_read_barrier_cache(rffi.SIGNEDP)
+            objint = llmemory.cast_adr_to_int(objadr)
+            assert WORD == 8, "check for 32bit compatibility"
+            index = (objint & StmGC.FX_MASK) / WORD
+            CP = lltype.Ptr(rffi.CArray(lltype.Signed))
+            rcp = rffi.cast(CP, read_cache[0])
+            if rcp[index] == objint:
+                return gcref_struct
+        else: # 'Q2R'
+            # is GCFLAG_PUBLIC_TO_PRIVATE or GCFLAG_MOVED set?
+            if not (objhdr.h_tid &
+                    (StmGC.GCFLAG_PUBLIC_TO_PRIVATE | StmGC.GCFLAG_MOVED)):
+                # no.
+                return gcref_struct
         
         funcptr = self.get_barrier_funcptr(returns_modified_object)
         res = funcptr(objadr)
@@ -462,9 +472,14 @@
         
 class STMWriteBarrierDescr(STMBarrierDescr):
     def __init__(self, gc_ll_descr, stmcat):
-        assert stmcat in ['P2W']
-        STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
-                                 'stm_WriteBarrier')
+        assert stmcat in ['A2W', 'V2W']
+        if stmcat == 'A2W':
+            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
+                                     'stm_WriteBarrier')
+        else:
+            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
+                                     'stm_RepeatWriteBarrier')
+
 
     @specialize.arg(2)
     def _do_barrier(self, gcref_struct, returns_modified_object):
@@ -473,9 +488,9 @@
         objadr = llmemory.cast_ptr_to_adr(gcref_struct)
         objhdr = rffi.cast(StmGC.GCHDRP, gcref_struct)
         
-        # if h_revision == privat_rev of transaction
+        # if it is a repeated WB or h_revision == privat_rev of transaction
         priv_rev = self.llop1.stm_get_adr_of_private_rev_num(rffi.SIGNEDP)
-        if objhdr.h_revision == priv_rev[0]:
+        if self.stmcat == 'V2W' or objhdr.h_revision == priv_rev[0]:
             # also WRITE_BARRIER not set?
             if not (objhdr.h_tid & StmGC.GCFLAG_WRITE_BARRIER):
                 return gcref_struct
@@ -580,8 +595,10 @@
 
     def _setup_write_barrier(self):
         if self.stm:
-            self.P2Rdescr = STMReadBarrierDescr(self, 'P2R')
-            self.P2Wdescr = STMWriteBarrierDescr(self, 'P2W')
+            self.A2Rdescr = STMReadBarrierDescr(self, 'A2R')
+            self.Q2Rdescr = STMReadBarrierDescr(self, 'Q2R')
+            self.A2Wdescr = STMWriteBarrierDescr(self, 'A2W')
+            self.V2Wdescr = STMWriteBarrierDescr(self, 'V2W')
             self.write_barrier_descr = "wbdescr: do not use"
         else:
             self.write_barrier_descr = WriteBarrierDescr(self)
diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -12,7 +12,7 @@
 #
 # Any SETFIELD_GC, SETARRAYITEM_GC, SETINTERIORFIELD_GC must be done on a
 # W object.  The operation that forces an object p1 to be W is
-# COND_CALL_STM_B(p1, descr=x2Wdescr), for x in 'PGORL'.  This
+# COND_CALL_STM_B(p1, descr=x2Wdescr), for x in 'AIQRVWZ'.  This
 # COND_CALL_STM_B is a bit special because if p1 is not W, it *replaces*
 # its value with the W copy (by changing the register's value and
 # patching the stack location if any).  It's still conceptually the same
@@ -25,6 +25,8 @@
 #
 
 
+
+
 class GcStmRewriterAssembler(GcRewriterAssembler):
     # This class performs the same rewrites as its base class,
     # plus the rewrites described above.
@@ -33,14 +35,7 @@
         GcRewriterAssembler.__init__(self, *args)
         self.known_category = {}    # variable: letter (R, W, ...)
         self.always_inevitable = False
-        self.more_precise_categories = {
-            'P': {'R': self.gc_ll_descr.P2Rdescr,
-                  'W': self.gc_ll_descr.P2Wdescr,
-                 },
-            'R': {'W': self.gc_ll_descr.P2Wdescr,
-                 },
-            'W': {},
-           }
+        
 
     def rewrite(self, operations):
         # overridden method from parent class
@@ -115,8 +110,6 @@
                 continue
             # ----------  mallocs  ----------
             if op.is_malloc():
-                # write barriers not valid after possible collection
-                self.write_to_read_categories()
                 self.handle_malloc_operation(op)
                 continue
             # ----------  calls  ----------
@@ -195,20 +188,45 @@
         assert not insert_transaction_break
         return self.newops
 
+    def emitting_an_operation_that_can_collect(self):
+        GcRewriterAssembler.emitting_an_operation_that_can_collect(self)
+        self.invalidate_write_categories()
+    
     def next_op_may_be_in_new_transaction(self):
         self.known_category.clear()
-        
-    def write_to_read_categories(self):
+
+    def invalidate_write_categories(self):
         for v, c in self.known_category.items():
             if c == 'W':
-                self.known_category[v] = 'R'
+                self.known_category[v] = 'V'
 
-    def clear_readable_statuses(self, reason):
+    def invalidate_read_categories(self, reason):
         # XXX: needs aliasing info to be better
         # XXX: move to optimizeopt to only invalidate same typed vars?
         for v, c in self.known_category.items():
             if c == 'R':
-                self.known_category[v] = 'P'
+                self.known_category[v] = 'Q'
+
+    
+    def get_barrier_descr(self, from_cat, to_cat):
+        # compare with translator.stm.funcgen.stm_barrier
+        # XXX: specialize more with info of IMMUTABLE and NOPTR
+        if from_cat >= to_cat:
+            return None
+        
+        gc = self.gc_ll_descr
+        if to_cat == 'W':
+            if from_cat >= 'V':
+                return gc.V2Wdescr
+            return gc.A2Wdescr
+        elif to_cat == 'V':
+            return gc.A2Wdescr
+        elif to_cat == 'R':
+            if from_cat >= 'Q':
+                return gc.Q2Rdescr
+            return gc.A2Rdescr
+        elif to_cat == 'I':
+            return gc.A2Rdescr
 
     def gen_initialize_tid(self, v_newgcobj, tid):
         GcRewriterAssembler.gen_initialize_tid(self, v_newgcobj, tid)
@@ -217,24 +235,23 @@
                               descr=self.gc_ll_descr.fielddescr_rev)
             self.newops.append(op)
             
-
-                
     def gen_write_barrier(self, v):
         raise NotImplementedError
 
     def gen_barrier(self, v_base, target_category):
         v_base = self.unconstifyptr(v_base)
         assert isinstance(v_base, BoxPtr)
-        source_category = self.known_category.get(v_base, 'P')
+        source_category = self.known_category.get(v_base, 'A')
+        write_barrier_descr = self.get_barrier_descr(source_category,
+                                                     target_category)
+        if write_barrier_descr is None:
+            return v_base    # no barrier needed
+
         if target_category == 'W':
             # if *any* of the readable vars is the same object,
             # it must repeat the read_barrier now
-            self.clear_readable_statuses(v_base)
-        mpcat = self.more_precise_categories[source_category]
-        try:
-            write_barrier_descr = mpcat[target_category]
-        except KeyError:
-            return v_base    # no barrier needed
+            self.invalidate_read_categories(v_base)
+
         args = [v_base,]
         op = rop.COND_CALL_STM_B
         self.newops.append(ResOperation(op, args, None,
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -93,7 +93,7 @@
             jump()
         """, """
             [p1, p2]
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setfield_gc(p1, p2, descr=tzdescr)
             stm_transaction_break(1)
             jump()
@@ -109,7 +109,7 @@
         """, """
             [p1, p2]
             p3 = same_as(ConstPtr(t))
-            cond_call_stm_b(p3, descr=P2Wdescr)
+            cond_call_stm_b(p3, descr=A2Wdescr)
             setfield_gc(p3, p2, descr=tzdescr)
             stm_transaction_break(1)
             jump()
@@ -130,11 +130,11 @@
             [p0]
             p1 = same_as(ConstPtr(t))
             p2 = same_as(ConstPtr(t))
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             p3 = getfield_gc(p1, descr=tzdescr)
-            cond_call_stm_b(p2, descr=P2Wdescr)
+            cond_call_stm_b(p2, descr=A2Wdescr)
             setfield_gc(p2, p0, descr=tzdescr)
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=Q2Rdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
             stm_transaction_break(1)
             jump()
@@ -153,11 +153,11 @@
             [p0]
             p1 = same_as(p0)
             p2 = same_as(p0)
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
-            cond_call_stm_b(p2, descr=P2Wdescr)
+            cond_call_stm_b(p2, descr=A2Wdescr)
             setfield_gc(p2, p0, descr=tzdescr)
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=Q2Rdescr)
             p5 = getfield_gc(p1, descr=tzdescr)
             stm_transaction_break(1)
             jump()
@@ -174,14 +174,14 @@
             jump()
         """, """
             [p0]
-            cond_call_stm_b(p0, descr=P2Rdescr)
+            cond_call_stm_b(p0, descr=A2Rdescr)
             p1 = getfield_gc(p0, descr=tzdescr)
             p2 = getfield_gc(p0, descr=tzdescr)
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             p3 = getfield_gc(p1, descr=tzdescr)
-            cond_call_stm_b(p2, descr=P2Wdescr)
+            cond_call_stm_b(p2, descr=A2Wdescr)
             setfield_gc(p2, p0, descr=tzdescr)
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=Q2Rdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
             stm_transaction_break(1)
             jump()
@@ -200,14 +200,14 @@
         """
         rewritten = """
             [p0, i1, i2]
-            cond_call_stm_b(p0, descr=P2Rdescr)
+            cond_call_stm_b(p0, descr=A2Rdescr)
             p1 = %s(p0, i1, descr=adescr)
             p2 = %s(p0, i2, descr=adescr)
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             p3 = getfield_gc(p1, descr=tzdescr)
-            cond_call_stm_b(p2, descr=P2Wdescr)
+            cond_call_stm_b(p2, descr=A2Wdescr)
             setfield_gc(p2, p0, descr=tzdescr)
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=Q2Rdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
             stm_transaction_break(1)
             jump()
@@ -225,12 +225,12 @@
             jump(p2)
         """, """
             [p1, p3]
-            cond_call_stm_b(p3, descr=P2Wdescr)
+            cond_call_stm_b(p3, descr=A2Wdescr)
             setfield_gc(p3, p1, descr=tzdescr)
             p2 = call_malloc_nursery(%(tdescr.size)d)
             setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr)
             stm_set_revision_gc(p2, descr=revdescr)
-            cond_call_stm_b(p3, descr=P2Wdescr)
+            cond_call_stm_b(p3, descr=V2Wdescr)
             setfield_gc(p3, p1, descr=tzdescr)
             stm_transaction_break(1)
             jump(p2)
@@ -245,7 +245,7 @@
             jump(p2)
         """, """
             [p1]
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
             p3 = call_malloc_nursery(%(tdescr.size)d)
             setfield_gc(p3, %(tdescr.tid)d, descr=tiddescr)
@@ -279,9 +279,9 @@
             jump()
         """, """
             [p1, p2, p3, p4]
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setfield_gc(p1, p2, descr=tzdescr)
-            cond_call_stm_b(p3, descr=P2Wdescr)
+            cond_call_stm_b(p3, descr=A2Wdescr)
             setfield_gc(p3, p4, descr=tzdescr)
             stm_transaction_break(1)
             jump()
@@ -295,7 +295,7 @@
             jump()
         """, """
             [p1, p2, i3]
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setfield_gc(p1, p2, descr=tzdescr)
             setfield_gc(p1, i3, descr=tydescr)
             stm_transaction_break(1)
@@ -311,10 +311,10 @@
             jump(p1)
         """, """
             [p1, p2, i3]
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setfield_gc(p1, p2, descr=tzdescr)
             label(p1, i3)
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setfield_gc(p1, i3, descr=tydescr)
             stm_transaction_break(1)
             jump(p1)
@@ -358,7 +358,7 @@
             jump(p2)
         """, """
             [p1]
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
             stm_transaction_break(1)
             jump(p2)
@@ -374,7 +374,7 @@
         """, """
             [p1]
             p3 = same_as(ConstPtr(t))
-            cond_call_stm_b(p3, descr=P2Rdescr)
+            cond_call_stm_b(p3, descr=A2Rdescr)
             p2 = getfield_gc(p3, descr=tzdescr)
             stm_transaction_break(1)
             jump(p2)
@@ -388,7 +388,7 @@
             jump(i3)
         """, """
             [p1, i2]
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             i3 = getarrayitem_gc(p1, i2, descr=adescr)
             stm_transaction_break(1)
             jump(i3)
@@ -401,7 +401,7 @@
             jump(i3)
         """, """
             [p1, i2]
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             i3 = getinteriorfield_gc(p1, i2, descr=adescr)
             stm_transaction_break(1)
             jump(i3)
@@ -415,7 +415,7 @@
             jump(p2, i2)
         """, """
             [p1]
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
             i2 = getfield_gc(p1, descr=tydescr)
             stm_transaction_break(1)
@@ -430,9 +430,9 @@
             jump(p2, i2)
         """, """
             [p1]
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
-            cond_call_stm_b(p2, descr=P2Rdescr)
+            cond_call_stm_b(p2, descr=A2Rdescr)
             i2 = getfield_gc(p2, descr=tydescr)
             stm_transaction_break(1)
             jump(p2, i2)
@@ -449,10 +449,10 @@
             jump(p1)
         """, """
             [p1]
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             i1 = getfield_gc(p1, descr=tydescr)
             i2 = int_add(i1, 1)
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setfield_gc(p1, i2, descr=tydescr)
             stm_transaction_break(1)
             jump(p1)
@@ -466,7 +466,7 @@
             jump(p2)
         """, """
             [p1]
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setfield_gc(p1, 123, descr=tydescr)
             p2 = getfield_gc(p1, descr=tzdescr)
             stm_transaction_break(1)
@@ -506,10 +506,10 @@
             jump(p2)
         """, """
             [p1]
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
             call(p2, descr=calldescr1)
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setfield_gc(p1, 5, descr=tydescr)
             stm_transaction_break(1)
             jump(p2)
@@ -589,9 +589,9 @@
             jump()
         """, """
             [p1, i1, p2, p3, i3, p4]
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setarrayitem_gc(p1, i1, p2, descr=adescr)
-            cond_call_stm_b(p3, descr=P2Wdescr)
+            cond_call_stm_b(p3, descr=A2Wdescr)
             setarrayitem_gc(p3, i3, p4, descr=adescr)
             stm_transaction_break(1)
             jump()
@@ -606,7 +606,7 @@
             jump()
         """, """
             [p1, p2, i2, p3, i3]
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setarrayitem_gc(p1, i2, p2, descr=adescr)
             i4 = read_timestamp()
             setarrayitem_gc(p1, i3, p3, descr=adescr)
@@ -623,7 +623,7 @@
             jump()
         """, """
             [p1, p2, i2, p3, i3]
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setinteriorfield_gc(p1, i2, p2, descr=adescr)
             i4 = read_timestamp()
             setinteriorfield_gc(p1, i3, p3, descr=adescr)
@@ -639,7 +639,7 @@
             jump()
         """, """
             [p1, i2, i3]
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             strsetitem(p1, i2, i3)
             unicodesetitem(p1, i2, i3)
             stm_transaction_break(1)
@@ -660,12 +660,12 @@
             jump(i2, p7)
         """, """
             [i1, i2, i3, p7]
-            cond_call_stm_b(p7, descr=P2Wdescr)
+            cond_call_stm_b(p7, descr=A2Wdescr)
             setfield_gc(p7, 10, descr=tydescr)
             call_release_gil(123, descr=calldescr2)
             guard_not_forced() []
             stm_transaction_break(0)
-            cond_call_stm_b(p7, descr=P2Wdescr)
+            cond_call_stm_b(p7, descr=A2Wdescr)
             setfield_gc(p7, 20, descr=tydescr)
             stm_transaction_break(1)
             jump(i2, p7)
@@ -689,11 +689,11 @@
                 jump(i2, p7)
             """ % op, """
                 [i1, i2, i3, p7]
-                cond_call_stm_b(p7, descr=P2Wdescr)
+                cond_call_stm_b(p7, descr=A2Wdescr)
                 setfield_gc(p7, 10, descr=tydescr)
                 $INEV
                 %s
-                cond_call_stm_b(p7, descr=P2Wdescr)
+                cond_call_stm_b(p7, descr=A2Wdescr)
                 setfield_gc(p7, 20, descr=tydescr)
                 stm_transaction_break(1)
                 jump(i2, p7)
@@ -706,8 +706,8 @@
             jump()
         """, """
             [p1, p2, i1, i2, i3]
-            cond_call_stm_b(p2, descr=P2Wdescr)
-            cond_call_stm_b(p1, descr=P2Rdescr)
+            cond_call_stm_b(p2, descr=A2Wdescr)
+            cond_call_stm_b(p1, descr=A2Rdescr)
             copystrcontent(p1, p2, i1, i2, i3)
             stm_transaction_break(1)
             jump()
@@ -727,7 +727,7 @@
                 jump(p1)
             """ % op, """
                 [p1]
-                cond_call_stm_b(p1, descr=P2Wdescr)
+                cond_call_stm_b(p1, descr=A2Wdescr)
                 setfield_gc(p1, 10, descr=tydescr)
                 %s
                 setfield_gc(p1, 20, descr=tydescr)
@@ -758,12 +758,12 @@
                 jump(p1)
             """ % (op, guard), """
                 [p1]
-                cond_call_stm_b(p1, descr=P2Wdescr)
+                cond_call_stm_b(p1, descr=A2Wdescr)
                 setfield_gc(p1, 10, descr=tydescr)
                 %s
                 %s
                 %s
-                cond_call_stm_b(p1, descr=P2Wdescr)
+                cond_call_stm_b(p1, descr=A2Wdescr)
                 setfield_gc(p1, 20, descr=tydescr)
                 stm_transaction_break(1)
                 jump(p1)
@@ -1141,7 +1141,7 @@
             stm_set_revision_gc(p1, descr=revdescr)
             setfield_gc(p1, 5, descr=clendescr)
             label(p1, i2, p3)
-            cond_call_stm_b(p1, descr=P2Wdescr)
+            cond_call_stm_b(p1, descr=A2Wdescr)
             setarrayitem_gc(p1, i2, p3, descr=cdescr)
         """)
 

From noreply at buildbot.pypy.org  Tue Oct 22 14:50:58 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Oct 2013 14:50:58 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: fix stm_integration_tests and start
 implementing repeated barriers
Message-ID: <20131022125058.E7F741C0403@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67513:a347f8e82fa4
Date: 2013-10-22 12:57 +0200
http://bitbucket.org/pypy/pypy/changeset/a347f8e82fa4/

Log:	fix stm_integration_tests and start implementing repeated barriers
	in assembler

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -81,7 +81,8 @@
             self.gc_size_of_header = WORD # for tests
         self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn)
         if gc_ll_descr.stm:
-            descrs = [gc_ll_descr.P2Rdescr, gc_ll_descr.P2Wdescr]
+            descrs = [gc_ll_descr.A2Rdescr, gc_ll_descr.Q2Rdescr,
+                      gc_ll_descr.A2Wdescr, gc_ll_descr.V2Wdescr]
         else:
             descrs = [gc_ll_descr.write_barrier_descr]
         for d in descrs:
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -189,6 +189,10 @@
         return mc.materialize(self.cpu.asmmemmgr, [])
 
     def _build_stm_transaction_break_path(self):
+        assert self.cpu.gc_ll_descr.stm
+        if not we_are_translated():
+            return    # tests only
+
         """ While arriving on slowpath, we have a gcpattern on stack 0.
         This function must preserve all registers
         """
@@ -1357,9 +1361,9 @@
             mc.MOV(ebp, mem(ecx, -WORD))
         #
         if gcrootmap and gcrootmap.is_stm:
-            if not hasattr(gc_ll_descr, 'P2Wdescr'):
+            if not hasattr(gc_ll_descr, 'A2Wdescr'):
                 raise Exception("unreachable code")
-            wbdescr = gc_ll_descr.P2Wdescr
+            wbdescr = gc_ll_descr.A2Wdescr
             self._stm_barrier_fastpath(mc, wbdescr, [ebp], is_frame=True,
                                        align_stack=align_stack)
             return
@@ -2486,43 +2490,48 @@
         #
         # FASTPATH:
         #
-        # write_barrier:
+        # A2W:
         # (obj->h_revision != stm_private_rev_num)
         #     || (obj->h_tid & GCFLAG_WRITE_BARRIER) != 0)
-        # read_barrier:
+        # V2W:
+        # (obj->h_tid & GCFLAG_WRITE_BARRIER) != 0)
+        # A2R:
         # (obj->h_revision != stm_private_rev_num)
         #     && (FXCACHE_AT(obj) != obj)))
+        # Q2R:
+        # (obj->h_tid & (GCFLAG_PUBLIC_TO_PRIVATE | GCFLAG_MOVED) != 0)
         if IS_X86_32:   # XXX: todo
             todo()
         jz_location = 0
         jz_location2 = 0
         jnz_location = 0
-        # compare h_revision with stm_private_rev_num (XXX: may be slow)
-        rn = self._get_stm_private_rev_num_addr()
-        if we_are_translated():
-            # during tests, _get_stm_private_rev_num_addr returns
-            # an absolute address, not a tl-offset
-            self._tl_segment_if_stm(mc)
-            mc.MOV_rj(X86_64_SCRATCH_REG.value, rn)
-        else: # testing:
-            mc.MOV(X86_64_SCRATCH_REG, heap(rn))
-            
-        if loc_base == ebp:
-            mc.CMP_rb(X86_64_SCRATCH_REG.value, StmGC.H_REVISION)
-        else:
-            mc.CMP(X86_64_SCRATCH_REG, mem(loc_base, StmGC.H_REVISION))
+        # compare h_revision with stm_private_rev_num
+        if descr.stmcat in ['A2W', 'A2R']:
+            rn = self._get_stm_private_rev_num_addr()
+            if we_are_translated():
+                # during tests, _get_stm_private_rev_num_addr returns
+                # an absolute address, not a tl-offset
+                self._tl_segment_if_stm(mc)
+                mc.MOV_rj(X86_64_SCRATCH_REG.value, rn)
+            else: # testing:
+                mc.MOV(X86_64_SCRATCH_REG, heap(rn))
+
+            if loc_base == ebp:
+                mc.CMP_rb(X86_64_SCRATCH_REG.value, StmGC.H_REVISION)
+            else:
+                mc.CMP(X86_64_SCRATCH_REG, mem(loc_base, StmGC.H_REVISION))
+            #
+            if descr.stmcat == 'A2R':
+                # jump to end if h_rev==priv_rev
+                mc.J_il8(rx86.Conditions['Z'], 0) # patched below
+                jz_location = mc.get_relative_pos()
+            else: # write_barrier
+                # jump to slowpath if h_rev!=priv_rev
+                mc.J_il8(rx86.Conditions['NZ'], 0) # patched below
+                jnz_location = mc.get_relative_pos()
         #
-        if descr.stmcat == 'P2R':#isinstance(descr, STMReadBarrierDescr):
-            # jump to end if h_rev==priv_rev
-            mc.J_il8(rx86.Conditions['Z'], 0) # patched below
-            jz_location = mc.get_relative_pos()
-        else: # write_barrier
-            # jump to slowpath if h_rev!=priv_rev
-            mc.J_il8(rx86.Conditions['NZ'], 0) # patched below
-            jnz_location = mc.get_relative_pos()
-
         # FXCACHE_AT(obj) != obj
-        if descr.stmcat == 'P2R':#isinstance(descr, STMReadBarrierDescr):
+        if descr.stmcat == 'A2R':
             # calculate: temp = obj & FX_MASK
             assert StmGC.FX_MASK == 65535
             assert not is_frame
@@ -2543,22 +2552,30 @@
             mc.CMP_rm(loc_base.value, (X86_64_SCRATCH_REG.value, 0))
             mc.J_il8(rx86.Conditions['Z'], 0) # patched below
             jz_location2 = mc.get_relative_pos()
+        #
+        # check flags:
+        if descr.stmcat in ['A2W', 'V2W', 'Q2R']:
+            if descr.stmcat in ['A2W', 'V2W']:
+                # obj->h_tid & GCFLAG_WRITE_BARRIER) != 0
+                assert IS_X86_64 and (StmGC.GCFLAG_WRITE_BARRIER >> 32) > 0
+                assert (StmGC.GCFLAG_WRITE_BARRIER >> 40) == 0
+                flags = StmGC.GCFLAG_WRITE_BARRIER >> 32
+            elif descr.stmcat == 'Q2R':
+                # obj->h_tid & PUBLIC_TO_PRIVATE|MOVED
+                flags = StmGC.GCFLAG_PUBLIC_TO_PRIVATE | StmGC.GCFLAG_MOVED
+                assert IS_X86_64 and (flags >> 32) > 0
+                assert (flags >> 40) == 0
+                flags = flags >> 32
 
-        # obj->h_tid & GCFLAG_WRITE_BARRIER) != 0
-        if descr.stmcat == 'P2W':#isinstance(descr, STMWriteBarrierDescr):
-            assert IS_X86_64 and (StmGC.GCFLAG_WRITE_BARRIER >> 32) > 0
-            assert (StmGC.GCFLAG_WRITE_BARRIER >> 40) == 0
             off = 4
-            flag = StmGC.GCFLAG_WRITE_BARRIER >> 32
             if loc_base == ebp:
-                mc.TEST8_bi(StmGC.H_TID + off, flag)
+                mc.TEST8_bi(StmGC.H_TID + off, flags)
             else:
-                mc.TEST8_mi((loc_base.value, StmGC.H_TID + off), flag)
+                mc.TEST8_mi((loc_base.value, StmGC.H_TID + off), flags)
 
             mc.J_il8(rx86.Conditions['Z'], 0) # patched below
             jz_location = mc.get_relative_pos()
-            # both conditions succeeded, jump to end
-            
+            # if flags not set, jump to end
             # jump target slowpath:
             offset = mc.get_relative_pos() - jnz_location
             assert 0 < offset <= 127
@@ -2592,7 +2609,7 @@
         offset = mc.get_relative_pos() - jz_location
         assert 0 < offset <= 127
         mc.overwrite(jz_location - 1, chr(offset))
-        if descr.stmcat == 'P2R':#isinstance(descr, STMReadBarrierDescr):
+        if descr.stmcat == 'A2R':#isinstance(descr, STMReadBarrierDescr):
             offset = mc.get_relative_pos() - jz_location2
             assert 0 < offset <= 127
             mc.overwrite(jz_location2 - 1, chr(offset))
diff --git a/rpython/jit/backend/x86/test/test_stm_integration.py b/rpython/jit/backend/x86/test/test_stm_integration.py
--- a/rpython/jit/backend/x86/test/test_stm_integration.py
+++ b/rpython/jit/backend/x86/test/test_stm_integration.py
@@ -152,8 +152,10 @@
             self.wb_called_on.append(obj)
             return obj
 
-        self.P2Rdescr = FakeSTMBarrier(self, 'P2R', read_barrier)
-        self.P2Wdescr = FakeSTMBarrier(self, 'P2W', write_barrier)
+        self.A2Rdescr = FakeSTMBarrier(self, 'A2R', read_barrier)
+        self.Q2Rdescr = FakeSTMBarrier(self, 'Q2R', read_barrier)
+        self.A2Wdescr = FakeSTMBarrier(self, 'A2W', write_barrier)
+        self.V2Wdescr = FakeSTMBarrier(self, 'V2W', write_barrier)
         
         self.do_write_barrier = None
         self.get_nursery_top_addr = None
@@ -253,8 +255,10 @@
                                                 cpu.__class__)
         
 
-        self.p2wd = cpu.gc_ll_descr.P2Wdescr
-        self.p2rd = cpu.gc_ll_descr.P2Rdescr
+        self.a2wd = cpu.gc_ll_descr.A2Wdescr
+        self.v2wd = cpu.gc_ll_descr.V2Wdescr
+        self.a2rd = cpu.gc_ll_descr.A2Rdescr
+        self.Q2rd = cpu.gc_ll_descr.Q2Rdescr
 
         TP = rffi.CArray(lltype.Signed)
         self.priv_rev_num = lltype.malloc(TP, 1, flavor='raw')
@@ -314,7 +318,7 @@
         
     def test_gc_read_barrier_fastpath(self):
         from rpython.jit.backend.llsupport.gc import STMReadBarrierDescr
-        descr = STMReadBarrierDescr(self.cpu.gc_ll_descr, 'P2R')
+        descr = STMReadBarrierDescr(self.cpu.gc_ll_descr, 'A2R')
 
         called = []
         def read(obj):
@@ -356,7 +360,7 @@
 
     def test_gc_write_barrier_fastpath(self):
         from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr
-        descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'P2W')
+        descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'A2W')
 
         called = []
         def write(obj):
@@ -416,7 +420,7 @@
             p0 = BoxPtr()
             operations = [
                 ResOperation(rop.COND_CALL_STM_B, [p0], None,
-                             descr=self.p2rd),
+                             descr=self.a2rd),
                 ResOperation(rop.FINISH, [p0], None, 
                              descr=BasicFinalDescr(0)),
                 ]
@@ -459,7 +463,7 @@
             p0 = BoxPtr()
             operations = [
                 ResOperation(rop.COND_CALL_STM_B, [p0], None,
-                             descr=self.p2wd),
+                             descr=self.a2wd),
                 ResOperation(rop.FINISH, [p0], None, 
                              descr=BasicFinalDescr(0)),
                 ]
@@ -787,9 +791,9 @@
             if llmemory.cast_ptr_to_adr(sgcref) == obj:
                 return rffi.cast(llmemory.Address, other_sgcref)
             return obj
-        P2W = FakeSTMBarrier(cpu.gc_ll_descr, 'P2W', write_barrier)
-        old_p2w = cpu.gc_ll_descr.P2Wdescr
-        cpu.gc_ll_descr.P2Wdescr = P2W
+        A2W = FakeSTMBarrier(cpu.gc_ll_descr, 'A2W', write_barrier)
+        old_a2w = cpu.gc_ll_descr.A2Wdescr
+        cpu.gc_ll_descr.A2Wdescr = A2W
 
         cpu.gc_ll_descr.init_nursery(100)
         cpu.setup_once()
@@ -801,10 +805,10 @@
         spill.initarglist([p0])
         operations = [
             ResOperation(rop.COND_CALL_STM_B, [p0], None,
-                         descr=P2W),
+                         descr=A2W),
             spill,
             ResOperation(rop.COND_CALL_STM_B, [p0], None,
-                         descr=P2W),
+                         descr=A2W),
             ResOperation(rop.FINISH, [p0], None, 
                              descr=BasicFinalDescr(0)),
             ]
@@ -818,7 +822,7 @@
         self.assert_in(called_on, [sgcref, other_sgcref])
 
         # for other tests:
-        cpu.gc_ll_descr.P2Wdescr = old_p2w
+        cpu.gc_ll_descr.A2Wdescr = old_a2w
 
     
         

From noreply at buildbot.pypy.org  Tue Oct 22 14:51:00 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Oct 2013 14:51:00 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: test and fix repeat_barriers
Message-ID: <20131022125100.1E2821C0403@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67514:c8762b1eb64d
Date: 2013-10-22 14:49 +0200
http://bitbucket.org/pypy/pypy/changeset/c8762b1eb64d/

Log:	test and fix repeat_barriers

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,3 +1,7 @@
+------------------------------------------------------------
+
+stm-jitdriver with autoreds
+
 ------------------------------------------------------------
 
 try to let non-atomic inevitable transactions run for longer, until
@@ -59,8 +63,4 @@
 * maybe GUARD_NOT_INEVITABLE after call_may_force, call_assembler
   which is a small check if we are inevitable and does a transaction_break
   if we are.
-** do not access thread-locals through thread_descriptor, but directly
-** have two versions of stm_transaction_break(1/2). One for after calls
-   which simply checks if the transaction is inevitable, and one to place
-   before JUMPs which calls stm_should_break_transaction()
 * look at XXXs for STM everywhere
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2555,6 +2555,7 @@
         #
         # check flags:
         if descr.stmcat in ['A2W', 'V2W', 'Q2R']:
+            flags = 0
             if descr.stmcat in ['A2W', 'V2W']:
                 # obj->h_tid & GCFLAG_WRITE_BARRIER) != 0
                 assert IS_X86_64 and (StmGC.GCFLAG_WRITE_BARRIER >> 32) > 0
@@ -2576,7 +2577,8 @@
             mc.J_il8(rx86.Conditions['Z'], 0) # patched below
             jz_location = mc.get_relative_pos()
             # if flags not set, jump to end
-            # jump target slowpath:
+        # jump target slowpath:
+        if jnz_location:
             offset = mc.get_relative_pos() - jnz_location
             assert 0 < offset <= 127
             mc.overwrite(jnz_location - 1, chr(offset))
diff --git a/rpython/jit/backend/x86/test/test_stm_integration.py b/rpython/jit/backend/x86/test/test_stm_integration.py
--- a/rpython/jit/backend/x86/test/test_stm_integration.py
+++ b/rpython/jit/backend/x86/test/test_stm_integration.py
@@ -258,7 +258,7 @@
         self.a2wd = cpu.gc_ll_descr.A2Wdescr
         self.v2wd = cpu.gc_ll_descr.V2Wdescr
         self.a2rd = cpu.gc_ll_descr.A2Rdescr
-        self.Q2rd = cpu.gc_ll_descr.Q2Rdescr
+        self.q2rd = cpu.gc_ll_descr.Q2Rdescr
 
         TP = rffi.CArray(lltype.Signed)
         self.priv_rev_num = lltype.malloc(TP, 1, flavor='raw')
@@ -358,6 +358,40 @@
             descr.llop1.set_cache_item(sgcref, 0)
 
 
+    def test_gc_repeat_read_barrier_fastpath(self):
+        from rpython.jit.backend.llsupport.gc import STMReadBarrierDescr
+        descr = STMReadBarrierDescr(self.cpu.gc_ll_descr, 'Q2R')
+
+        called = []
+        def read(obj):
+            called.append(obj)
+            return obj
+
+        functype = lltype.Ptr(lltype.FuncType(
+            [llmemory.Address], llmemory.Address))
+        funcptr = llhelper(functype, read)
+        descr.b_failing_case_ptr = funcptr
+        descr.llop1 = fakellop()
+
+        # -------- TEST --------
+        for flags in [StmGC.GCFLAG_PUBLIC_TO_PRIVATE|StmGC.GCFLAG_MOVED, 0]:
+            called[:] = []
+            
+            s = self.allocate_prebuilt_s()
+            sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+            s.h_tid |= flags
+
+            descr._do_barrier(sgcref,
+                              returns_modified_object=True)
+                        
+            # check if rev-fastpath worked
+            if not flags:
+                # fastpath
+                self.assert_not_in(called, [sgcref])
+            else:
+                self.assert_in(called, [sgcref])
+
+
     def test_gc_write_barrier_fastpath(self):
         from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr
         descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'A2W')
@@ -384,7 +418,7 @@
             descr._do_barrier(sgcref,
                               returns_modified_object=True)
                         
-            # check if rev-fastpath worked
+            # check if fastpath worked
             if rev == fakellop.PRIV_REV:
                 # fastpath
                 self.assert_not_in(called, [sgcref])
@@ -398,6 +432,36 @@
                               returns_modified_object=True)
             self.assert_in(called, [sgcref])
 
+    def test_gc_repeat_write_barrier_fastpath(self):
+        from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr
+        descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'V2W')
+
+        called = []
+        def write(obj):
+            called.append(obj)
+            return obj
+
+        functype = lltype.Ptr(lltype.FuncType(
+            [llmemory.Address], llmemory.Address))
+        funcptr = llhelper(functype, write)
+        descr.b_failing_case_ptr = funcptr
+        descr.llop1 = fakellop()
+
+        # -------- TEST --------
+        s = self.allocate_prebuilt_s()
+        sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+
+        descr._do_barrier(sgcref,
+                          returns_modified_object=True)
+                        
+        # fastpath (WRITE_BARRIER not set)
+        self.assert_not_in(called, [sgcref])
+
+        # now set WRITE_BARRIER -> always call slowpath
+        s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER
+        descr._do_barrier(sgcref, 
+            returns_modified_object=True)
+        self.assert_in(called, [sgcref])
         
         
         
@@ -445,6 +509,40 @@
             # not called:
             assert not called_on
 
+    def test_repeat_read_barrier_fastpath(self):
+        cpu = self.cpu
+        cpu.gc_ll_descr.init_nursery(100)
+        cpu.setup_once()
+
+        called_on = cpu.gc_ll_descr.rb_called_on
+        for flags in [StmGC.GCFLAG_PUBLIC_TO_PRIVATE|StmGC.GCFLAG_MOVED, 0]:
+            cpu.gc_ll_descr.clear_lists()
+            self.clear_read_cache()
+            
+            s = self.allocate_prebuilt_s()
+            sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+            s.h_tid |= flags
+            
+            p0 = BoxPtr()
+            operations = [
+                ResOperation(rop.COND_CALL_STM_B, [p0], None,
+                             descr=self.q2rd),
+                ResOperation(rop.FINISH, [p0], None, 
+                             descr=BasicFinalDescr(0)),
+                ]
+            inputargs = [p0]
+            looptoken = JitCellToken()
+            cpu.compile_loop(None, inputargs, operations, looptoken)
+            self.cpu.execute_token(looptoken, sgcref)
+            
+            # check if rev-fastpath worked
+            if not flags:
+                # fastpath
+                self.assert_not_in(called_on, [sgcref])
+            else:
+                self.assert_in(called_on, [sgcref])
+
+
     def test_write_barrier_fastpath(self):
         cpu = self.cpu
         cpu.gc_ll_descr.init_nursery(100)
@@ -485,6 +583,39 @@
             self.cpu.execute_token(looptoken, sgcref)
             self.assert_in(called_on, [sgcref])
 
+    def test_repeat_write_barrier_fastpath(self):
+        cpu = self.cpu
+        cpu.gc_ll_descr.init_nursery(100)
+        cpu.setup_once()
+
+        called_on = cpu.gc_ll_descr.wb_called_on
+        cpu.gc_ll_descr.clear_lists()
+            
+        s = self.allocate_prebuilt_s()
+        sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+
+        p0 = BoxPtr()
+        operations = [
+            ResOperation(rop.COND_CALL_STM_B, [p0], None,
+                         descr=self.v2wd),
+            ResOperation(rop.FINISH, [p0], None, 
+                        descr=BasicFinalDescr(0)),
+            ]
+        
+        inputargs = [p0]
+        looptoken = JitCellToken()
+        cpu.compile_loop(None, inputargs, operations, looptoken)
+        self.cpu.execute_token(looptoken, sgcref)
+            
+        # fastpath and WRITE_BARRIER not set
+        self.assert_not_in(called_on, [sgcref])
+
+        # now set WRITE_BARRIER -> always call slowpath
+        cpu.gc_ll_descr.clear_lists()
+        s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER
+        self.cpu.execute_token(looptoken, sgcref)
+        self.assert_in(called_on, [sgcref])
+
             
     def test_ptr_eq_fastpath(self):
         cpu = self.cpu

From noreply at buildbot.pypy.org  Tue Oct 22 15:16:33 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Oct 2013 15:16:33 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: Start by
	adjusting the comment
Message-ID: <20131022131633.AE6C11C2FFF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67515:02dac9fc5132
Date: 2013-10-22 15:15 +0200
http://bitbucket.org/pypy/pypy/changeset/02dac9fc5132/

Log:	Start by adjusting the comment

diff --git a/rpython/rtyper/lltypesystem/rlist.py b/rpython/rtyper/lltypesystem/rlist.py
--- a/rpython/rtyper/lltypesystem/rlist.py
+++ b/rpython/rtyper/lltypesystem/rlist.py
@@ -13,20 +13,15 @@
 
 # ____________________________________________________________
 #
-#  Concrete implementation of RPython lists:
+#  The concrete implementation of resized RPython lists is as a GcStruct
+#  with only one field: a pointer to an overallocated array of items.
+#  This overallocated array is a C-like array in memory preceded by
+#  three fields: the GC header, 'allocated_length', and 'used_length'.
+#  In the array part, each item contains a primitive value or pointer
+#  to the actual list item.
 #
-#    struct list {
-#        int length;
-#        items_array *items;
-#    }
-#
-#    'items' points to a C-like array in memory preceded by a 'length' header,
-#    where each item contains a primitive value or pointer to the actual list
-#    item.
-#
-#    or for fixed-size lists an array is directly used:
-#
-#    item_t list_items[]
+#  For fixed-size lists, we just use a GcArray, which has only one
+#  'length' after the GC header.
 #
 
 class BaseListRepr(AbstractBaseListRepr):

From noreply at buildbot.pypy.org  Tue Oct 22 17:36:06 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 22 Oct 2013 17:36:06 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Remove a bad
	assertion
Message-ID: <20131022153606.85B611C0144@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-array_prepare_-array_wrap
Changeset: r67516:b91225187b3a
Date: 2013-10-22 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/b91225187b3a/

Log:	Remove a bad assertion

diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -240,7 +240,6 @@
         return res
 
     def call_prepare(self, space, w_out, w_obj, w_result):
-        assert isinstance(w_result, W_NDimArray)
         if isinstance(w_out, W_NDimArray):
             w_array = space.lookup(w_out, "__array_prepare__")
             w_caller = w_out

From noreply at buildbot.pypy.org  Tue Oct 22 17:36:07 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 22 Oct 2013 17:36:07 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Fix
	FakeObjSpace.lookup
Message-ID: <20131022153607.EBC401C2FF1@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-array_prepare_-array_wrap
Changeset: r67517:f473599db4d8
Date: 2013-10-22 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/f473599db4d8/

Log:	Fix FakeObjSpace.lookup

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -208,8 +208,7 @@
             return self.w_None
 
     def lookup(self, w_obj, name):
-        w_type = self.type(w_obj)
-        return w_type.lookup(name)
+        return w_obj.getdictvalue(self, name)
 
     def gettypefor(self, w_obj):
         return None

From noreply at buildbot.pypy.org  Tue Oct 22 18:02:26 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 22 Oct 2013 18:02:26 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Add Box.dtype
Message-ID: <20131022160226.90A7A1C0149@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-array_prepare_-array_wrap
Changeset: r67518:7d623144a86e
Date: 2013-10-22 18:01 +0200
http://bitbucket.org/pypy/pypy/changeset/7d623144a86e/

Log:	Add Box.dtype

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -251,6 +251,9 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "view not implelemnted yet"))
 
+    def descr_dtype(self, space):
+        return self._get_dtype(space)
+
 class W_BoolBox(W_GenericBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
@@ -524,6 +527,7 @@
     round = interp2app(W_GenericBox.descr_round),
     conjugate = interp2app(W_GenericBox.descr_conjugate),
     view = interp2app(W_GenericBox.descr_view),
+    dtype = GetSetProperty(W_GenericBox.descr_dtype)
 )
 
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,

From noreply at buildbot.pypy.org  Tue Oct 22 18:49:50 2013
From: noreply at buildbot.pypy.org (lwassermann)
Date: Tue, 22 Oct 2013 18:49:50 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: removed tracing on keyboard
	's' and 't'
Message-ID: <20131022164950.E4DB51C0149@cobra.cs.uni-duesseldorf.de>

Author: Lars Wassermann <lars.wassermann at gmail.com>
Branch: 
Changeset: r514:3494932a8f0d
Date: 2013-10-22 18:49 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/3494932a8f0d/

Log:	removed tracing on keyboard 's' and 't'

diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -754,7 +754,7 @@
     # This test deliberately test for equal W_Object class. The Smalltalk classes
     # might be different (e.g. Symbol and ByteString)
     if w_rcvr.__class__ is not w_replacement.__class__:
-        raise PrimitiveFailedError()
+        raise PrimitiveFailedError
     if (w_rcvr.size() - w_rcvr.instsize(interp.space) <= stop
             or w_replacement.size() - w_replacement.instsize(interp.space) <= repStart + (stop - start)):
         raise PrimitiveFailedError()
@@ -783,10 +783,6 @@
 @expose_primitive(KBD_NEXT, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
     code = interp.space.get_display().next_keycode()
-    if code == ord('t'):
-        interp.trace = True
-    elif code == ord('s'):
-        interp.trace = False
     if code == 0:
         return interp.space.w_nil
     else:
diff --git a/spyvm/shadow.py b/spyvm/shadow.py
--- a/spyvm/shadow.py
+++ b/spyvm/shadow.py
@@ -1425,4 +1425,4 @@
     #         w_self = self.w_self()
     #         assert isinstance(w_self, model.W_PointersObject)
     #         w_self._shadow = None
-    #         raise error.PrimitiveFailedError
\ No newline at end of file
+    #         raise error.PrimitiveFailedError

From noreply at buildbot.pypy.org  Tue Oct 22 19:05:04 2013
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 22 Oct 2013 19:05:04 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-array_prepare_-array_wrap: Merge default
	(with pain)
Message-ID: <20131022170504.F21BF1C0149@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: numpypy-array_prepare_-array_wrap
Changeset: r67519:98446563b964
Date: 2013-10-22 19:04 +0200
http://bitbucket.org/pypy/pypy/changeset/98446563b964/

Log:	Merge default (with pain)

diff too long, truncating to 2000 out of 12471 lines

diff --git a/lib_pypy/numpy.py b/lib_pypy/numpy.py
deleted file mode 100644
--- a/lib_pypy/numpy.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import warnings
-import sys
-if 'numpypy' not in sys.modules:
-    warnings.warn(
-        "The 'numpy' module of PyPy is in-development and not complete. "
-        "To avoid this warning, write 'import numpypy as numpy'. ",
-        UserWarning) # XXX is this the best warning type?
-
-from numpypy import *
-import numpypy
-__all__ = numpypy.__all__
-del numpypy
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,51 +1,17 @@
-import core
-from core import *
-import lib
-from lib import *
+from . import core
+from .core import *
+from . import lib
+from .lib import *
 
 from __builtin__ import bool, int, long, float, complex, object, unicode, str
-from core import abs, max, min
+
+from .core import round, abs, max, min
 
 __version__ = '1.7.0'
 
-import os
-def get_include():
-    """
-    Return the directory that contains the NumPy \\*.h header files.
-
-    Extension modules that need to compile against NumPy should use this
-    function to locate the appropriate include directory.
-
-    Notes
-    -----
-    When using ``distutils``, for example in ``setup.py``.
-    ::
-
-        import numpy as np
-        ...
-        Extension('extension_name', ...
-                include_dirs=[np.get_include()])
-        ...
-
-    """
-    import numpy
-    if getattr(numpy, 'show_config', None) is None:
-        # running from numpy source directory
-        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-        return os.path.join(head, '../include')
-    else:
-        # using installed numpy core headers
-        import numpy.core as core
-        d = os.path.join(os.path.dirname(core.__file__), 'include')
-    return d
-
-
-
-__all__ = ['__version__', 'get_include']
+__all__ = ['__version__']
 __all__ += core.__all__
 __all__ += lib.__all__
 
 #import sys
 #sys.modules.setdefault('numpy', sys.modules['numpypy'])
-
-
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
--- a/lib_pypy/numpypy/core/__init__.py
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -1,12 +1,17 @@
-import numeric
-from numeric import *
-import fromnumeric
-from fromnumeric import *
-import shape_base
-from shape_base import *
+from __future__ import division, absolute_import, print_function
 
-from fromnumeric import amax as max, amin as min
-from numeric import absolute as abs
+from . import multiarray
+from . import umath
+from . import numeric
+from .numeric import *
+from . import fromnumeric
+from .fromnumeric import *
+from . import shape_base
+from .shape_base import *
+
+from .fromnumeric import amax as max, amin as min, \
+    round_ as round
+from .numeric import absolute as abs
 
 __all__ = []
 __all__ += numeric.__all__
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ b/lib_pypy/numpypy/core/arrayprint.py
@@ -247,10 +247,11 @@
     formatdict = {'bool' : _boolFormatter,
                   'int' : IntegerFormat(data),
                   'float' : FloatFormat(data, precision, suppress_small),
-                  'longfloat' : LongFloatFormat(precision),
+                  'longfloat' : FloatFormat(data, precision, suppress_small),
                   'complexfloat' : ComplexFormat(data, precision,
                                                  suppress_small),
-                  'longcomplexfloat' : LongComplexFormat(precision),
+                  'longcomplexfloat' : ComplexFormat(data, precision,
+                                                     suppress_small),
                   'datetime' : DatetimeFormat(data),
                   'timedelta' : TimedeltaFormat(data),
                   'numpystr' : repr_format,
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -1,36 +1,53 @@
-######################################################################    
+######################################################################
 # This is a copy of numpy/core/fromnumeric.py modified for numpypy
 ######################################################################
-# Each name in __all__ was a function in  'numeric' that is now 
-# a method in 'numpy'.
-# When the corresponding method is added to numpypy BaseArray
-# each function should be added as a module function 
-# at the applevel 
-# This can be as simple as doing the following
-#
-# def func(a, ...):
-#     if not hasattr(a, 'func')
-#         a = numpypy.array(a)
-#     return a.func(...)
-#
-######################################################################
-
-import numpypy
-import _numpypy
-
-# Module containing non-deprecated functions borrowed from Numeric.
-__docformat__ = "restructuredtext en"
+"""Module containing non-deprecated functions borrowed from Numeric.
+
+"""
+from __future__ import division, absolute_import, print_function
+
+import types
+
+from . import multiarray as mu
+from . import umath as um
+from . import numerictypes as nt
+from .numeric import asarray, array, asanyarray, concatenate
+from . import _methods
+
+
+# functions that are methods
+__all__ = [
+        'alen', 'all', 'alltrue', 'amax', 'amin', 'any', 'argmax',
+        'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip',
+        'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean',
+        'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put',
+        'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_',
+        'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze',
+        'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var',
+        ]
+
+
+try:
+    _gentype = types.GeneratorType
+except AttributeError:
+    _gentype = type(None)
+
+# save away Python sum
+_sum_ = sum
 
 # functions that are now methods
-__all__ = ['take', 'reshape', 'choose', 'repeat', 'put',
-           'swapaxes', 'transpose', 'sort', 'argsort', 'argmax', 'argmin',
-           'searchsorted', 'alen',
-           'resize', 'diagonal', 'trace', 'ravel', 'nonzero', 'shape',
-           'compress', 'clip', 'sum', 'product', 'prod', 'sometrue', 'alltrue',
-           'any', 'all', 'cumsum', 'cumproduct', 'cumprod', 'ptp', 'ndim',
-           'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze',
-           'amax', 'amin',
-          ]
+def _wrapit(obj, method, *args, **kwds):
+    try:
+        wrap = obj.__array_wrap__
+    except AttributeError:
+        wrap = None
+    result = getattr(asarray(obj), method)(*args, **kwds)
+    if wrap:
+        if not isinstance(result, mu.ndarray):
+            result = asarray(result)
+        result = wrap(result)
+    return result
+
 
 def take(a, indices, axis=None, out=None, mode='raise'):
     """
@@ -46,6 +63,10 @@
         The source array.
     indices : array_like
         The indices of the values to extract.
+
+        .. versionadded:: 1.8.0
+
+        Also allow scalars for indices.
     axis : int, optional
         The axis over which to select values. By default, the flattened
         input array is used.
@@ -85,8 +106,17 @@
     >>> a[indices]
     array([4, 3, 6])
 
+    If `indices` is not one dimensional, the output also has these dimensions.
+
+    >>> np.take(a, [[0, 1], [2, 3]])
+    array([[4, 3],
+           [5, 7]])
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        take = a.take
+    except AttributeError:
+        return _wrapit(a, 'take', indices, axis, out, mode)
+    return take(indices, axis, out, mode)
 
 
 # not deprecated --- copy if necessary, view otherwise
@@ -104,16 +134,23 @@
         One shape dimension can be -1. In this case, the value is inferred
         from the length of the array and remaining dimensions.
     order : {'C', 'F', 'A'}, optional
-        Determines whether the array data should be viewed as in C
-        (row-major) order, FORTRAN (column-major) order, or the C/FORTRAN
-        order should be preserved.
+        Read the elements of `a` using this index order, and place the elements
+        into the reshaped array using this index order.  'C' means to
+        read / write the elements using C-like index order, with the last axis index
+        changing fastest, back to the first axis index changing slowest.  'F'
+        means to read / write the elements using Fortran-like index order, with
+        the first index changing fastest, and the last index changing slowest.
+        Note that the 'C' and 'F' options take no account of the memory layout
+        of the underlying array, and only refer to the order of indexing.  'A'
+        means to read / write the elements in Fortran-like index order if `a` is
+        Fortran *contiguous* in memory, C-like order otherwise.
 
     Returns
     -------
     reshaped_array : ndarray
         This will be a new view object if possible; otherwise, it will
-        be a copy.
-
+        be a copy.  Note there is no guarantee of the *memory layout* (C- or
+        Fortran- contiguous) of the returned array.
 
     See Also
     --------
@@ -121,7 +158,6 @@
 
     Notes
     -----
-
     It is not always possible to change the shape of an array without
     copying the data. If you want an error to be raise if the data is copied,
     you should assign the new shape to the shape attribute of the array::
@@ -129,12 +165,39 @@
      >>> a = np.zeros((10, 2))
      # A transpose make the array non-contiguous
      >>> b = a.T
-     # Taking a view makes it possible to modify the shape without modiying the
+     # Taking a view makes it possible to modify the shape without modifying the
      # initial object.
      >>> c = b.view()
      >>> c.shape = (20)
      AttributeError: incompatible shape for a non-contiguous array
 
+    The `order` keyword gives the index ordering both for *fetching* the values
+    from `a`, and then *placing* the values into the output array.  For example,
+    let's say you have an array:
+
+    >>> a = np.arange(6).reshape((3, 2))
+    >>> a
+    array([[0, 1],
+           [2, 3],
+           [4, 5]])
+
+    You can think of reshaping as first raveling the array (using the given
+    index order), then inserting the elements from the raveled array into the
+    new array using the same kind of index ordering as was used for the
+    raveling.
+
+    >>> np.reshape(a, (2, 3)) # C-like index ordering
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.reshape(a, (2, 3), order='F') # Fortran-like index ordering
+    array([[0, 4, 3],
+           [2, 1, 5]])
+    >>> np.reshape(np.ravel(a, order='F'), (2, 3), order='F')
+    array([[0, 4, 3],
+           [2, 1, 5]])
 
     Examples
     --------
@@ -148,12 +211,13 @@
     array([[1, 2],
            [3, 4],
            [5, 6]])
-
     """
     assert order == 'C'
-    if not hasattr(a, 'reshape'):
-       a = numpypy.array(a)
-    return a.reshape(newshape)
+    try:
+        reshape = a.reshape
+    except AttributeError:
+        return _wrapit(a, 'reshape', newshape)
+    return reshape(newshape)
 
 
 def choose(a, choices, out=None, mode='raise'):
@@ -275,7 +339,11 @@
             [-1, -2, -3, -4, -5]]])
 
     """
-    return _numpypy.choose(a, choices, out, mode)
+    try:
+        choose = a.choose
+    except AttributeError:
+        return _wrapit(a, 'choose', choices, out=out, mode=mode)
+    return choose(choices, out=out, mode=mode)
 
 
 def repeat(a, repeats, axis=None):
@@ -317,7 +385,11 @@
            [3, 4]])
 
     """
-    return _numpypy.repeat(a, repeats, axis)
+    try:
+        repeat = a.repeat
+    except AttributeError:
+        return _wrapit(a, 'repeat', repeats, axis)
+    return repeat(repeats, axis)
 
 
 def put(a, ind, v, mode='raise'):
@@ -368,7 +440,7 @@
     array([ 0,  1,  2,  3, -5])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return a.put(ind, v, mode)
 
 
 def swapaxes(a, axis1, axis2):
@@ -412,7 +484,10 @@
             [3, 7]]])
 
     """
-    swapaxes = a.swapaxes
+    try:
+        swapaxes = a.swapaxes
+    except AttributeError:
+        return _wrapit(a, 'swapaxes', axis1, axis2)
     return swapaxes(axis1, axis2)
 
 
@@ -456,9 +531,158 @@
     """
     if axes is not None:
         raise NotImplementedError('No "axes" arg yet.')
-    if not hasattr(a, 'T'):
-        a = numpypy.array(a)
-    return a.T
+    try:
+        transpose = a.transpose
+    except AttributeError:
+        return _wrapit(a, 'transpose')
+    return transpose()
+
+
+def partition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Return a partitioned copy of an array.
+
+    Creates a copy of the array with its elements rearranged in such a way that
+    the value of the element in kth position is in the position it would be in
+    a sorted array. All elements smaller than the kth element are moved before
+    this element and all equal or greater are moved behind it. The ordering of
+    the elements in the two partitions is undefined.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to be sorted.
+    kth : int or sequence of ints
+        Element index to partition by. The kth value of the element will be in
+        its final sorted position and all smaller elements will be moved before
+        it and all equal or greater elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all elements
+        indexed by kth  of them into their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort. If None, the array is flattened before
+        sorting. The default is -1, which sorts along the last axis.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'.
+    order : list, optional
+        When `a` is a structured array, this argument specifies which fields
+        to compare first, second, and so on.  This list does not need to
+        include all of the fields.
+
+    Returns
+    -------
+    partitioned_array : ndarray
+        Array of the same type and shape as `a`.
+
+    See Also
+    --------
+    ndarray.partition : Method to sort an array in-place.
+    argpartition : Indirect partition.
+    sort : Full sorting
+
+    Notes
+    -----
+    The various selection algorithms are characterized by their average speed,
+    worst case performance, work space size, and whether they are stable. A
+    stable sort keeps items with the same key in the same relative order. The
+    three available algorithms have the following properties:
+
+    ================= ======= ============= ============ =======
+       kind            speed   worst case    work space  stable
+    ================= ======= ============= ============ =======
+    'introselect'        1        O(n)           0         no
+    ================= ======= ============= ============ =======
+
+    All the partition algorithms make temporary copies of the data when
+    partitioning along any but the last axis.  Consequently, partitioning
+    along the last axis is faster and uses less space than partitioning
+    along any other axis.
+
+    The sort order for complex numbers is lexicographic. If both the real
+    and imaginary parts are non-nan then the order is determined by the
+    real parts except when they are equal, in which case the order is
+    determined by the imaginary parts.
+
+    Examples
+    --------
+    >>> a = np.array([3, 4, 2, 1])
+    >>> np.partition(a, 3)
+    array([2, 1, 3, 4])
+
+    >>> np.partition(a, (1, 3))
+    array([1, 2, 3, 4])
+
+    """
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.partition(kth, axis=axis, kind=kind, order=order)
+    return a
+
+
+def argpartition(a, kth, axis=-1, kind='introselect', order=None):
+    """
+    Perform an indirect partition along the given axis using the algorithm
+    specified by the `kind` keyword. It returns an array of indices of the
+    same shape as `a` that index data along the given axis in partitioned
+    order.
+
+    .. versionadded:: 1.8.0
+
+    Parameters
+    ----------
+    a : array_like
+        Array to sort.
+    kth : int or sequence of ints
+        Element index to partition by. The kth element will be in its final
+        sorted position and all smaller elements will be moved before it and
+        all larger elements behind it.
+        The order all elements in the partitions is undefined.
+        If provided with a sequence of kth it will partition all of them into
+        their sorted position at once.
+    axis : int or None, optional
+        Axis along which to sort.  The default is -1 (the last axis). If None,
+        the flattened array is used.
+    kind : {'introselect'}, optional
+        Selection algorithm. Default is 'introselect'
+    order : list, optional
+        When `a` is an array with fields defined, this argument specifies
+        which fields to compare first, second, etc.  Not all fields need be
+        specified.
+
+    Returns
+    -------
+    index_array : ndarray, int
+        Array of indices that partition `a` along the specified axis.
+        In other words, ``a[index_array]`` yields a sorted `a`.
+
+    See Also
+    --------
+    partition : Describes partition algorithms used.
+    ndarray.partition : Inplace partition.
+    argsort : Full indirect sort
+
+    Notes
+    -----
+    See `partition` for notes on the different selection algorithms.
+
+    Examples
+    --------
+    One dimensional array:
+
+    >>> x = np.array([3, 4, 2, 1])
+    >>> x[np.argpartition(x, 3)]
+    array([2, 1, 3, 4])
+    >>> x[np.argpartition(x, (1, 3))]
+    array([1, 2, 3, 4])
+
+    """
+    return a.argpartition(kth, axis, kind=kind, order=order)
+
 
 def sort(a, axis=-1, kind='quicksort', order=None):
     """
@@ -489,6 +713,7 @@
     argsort : Indirect sort.
     lexsort : Indirect stable sort on multiple keys.
     searchsorted : Find elements in a sorted array.
+    partition : Partial sort.
 
     Notes
     -----
@@ -559,7 +784,13 @@
           dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    if axis is None:
+        a = asanyarray(a).flatten()
+        axis = 0
+    else:
+        a = asanyarray(a).copy()
+    a.sort(axis, kind, order)
+    return a
 
 
 def argsort(a, axis=-1, kind='quicksort', order=None):
@@ -595,6 +826,7 @@
     sort : Describes sorting algorithms used.
     lexsort : Indirect stable sort with multiple keys.
     ndarray.sort : Inplace sort.
+    argpartition : Indirect partial sort.
 
     Notes
     -----
@@ -640,7 +872,11 @@
     array([0, 1])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        argsort = a.argsort
+    except AttributeError:
+        return _wrapit(a, 'argsort', axis, kind, order)
+    return argsort(axis, kind, order)
 
 
 def argmax(a, axis=None):
@@ -694,9 +930,11 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmax'):
-        a = numpypy.array(a)
-    return a.argmax()
+    try:
+        argmax = a.argmax
+    except AttributeError:
+        return _wrapit(a, 'argmax')
+    return argmax()
 
 
 def argmin(a, axis=None):
@@ -710,29 +948,37 @@
 
     """
     assert axis is None
-    if not hasattr(a, 'argmin'):
-        a = numpypy.array(a)
-    return a.argmin()
-
-
-def searchsorted(a, v, side='left'):
+    try:
+        argmin = a.argmin
+    except AttributeError:
+        return _wrapit(a, 'argmin')
+    return argmin()
+
+
+def searchsorted(a, v, side='left', sorter=None):
     """
     Find indices where elements should be inserted to maintain order.
 
-    Find the indices into a sorted array `a` such that, if the corresponding
-    elements in `v` were inserted before the indices, the order of `a` would
-    be preserved.
+    Find the indices into a sorted array `a` such that, if the
+    corresponding elements in `v` were inserted before the indices, the
+    order of `a` would be preserved.
 
     Parameters
     ----------
     a : 1-D array_like
-        Input array, sorted in ascending order.
+        Input array. If `sorter` is None, then it must be sorted in
+        ascending order, otherwise `sorter` must be an array of indices
+        that sort it.
     v : array_like
         Values to insert into `a`.
     side : {'left', 'right'}, optional
-        If 'left', the index of the first suitable location found is given.  If
-        'right', return the last such index.  If there is no suitable
+        If 'left', the index of the first suitable location found is given.
+        If 'right', return the last such index.  If there is no suitable
         index, return either 0 or N (where N is the length of `a`).
+    sorter : 1-D array_like, optional
+        .. versionadded:: 1.7.0
+        Optional array of integer indices that sort array a into ascending
+        order. They are typically the result of argsort.
 
     Returns
     -------
@@ -761,7 +1007,11 @@
     array([0, 5, 1, 2])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        searchsorted = a.searchsorted
+    except AttributeError:
+        return _wrapit(a, 'searchsorted', v, side, sorter)
+    return searchsorted(v, side, sorter)
 
 
 def resize(a, new_shape):
@@ -802,10 +1052,30 @@
            [0, 1, 2, 3]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def squeeze(a):
+    if isinstance(new_shape, (int, nt.integer)):
+        new_shape = (new_shape,)
+    a = ravel(a)
+    Na = len(a)
+    if not Na: return mu.zeros(new_shape, a.dtype.char)
+    total_size = um.multiply.reduce(new_shape)
+    n_copies = int(total_size / Na)
+    extra = total_size % Na
+
+    if total_size == 0:
+        return a[:0]
+
+    if extra != 0:
+        n_copies = n_copies+1
+        extra = Na-extra
+
+    a = concatenate( (a,)*n_copies)
+    if extra > 0:
+        a = a[:-extra]
+
+    return reshape(a, new_shape)
+
+
+def squeeze(a, axis=None):
     """
     Remove single-dimensional entries from the shape of an array.
 
@@ -813,12 +1083,19 @@
     ----------
     a : array_like
         Input data.
+    axis : None or int or tuple of ints, optional
+        .. versionadded:: 1.7.0
+
+        Selects a subset of the single-dimensional entries in the
+        shape. If an axis is selected with shape entry greater than
+        one, an error is raised.
 
     Returns
     -------
     squeezed : ndarray
-        The input array, but with with all dimensions of length 1
-        removed.  Whenever possible, a view on `a` is returned.
+        The input array, but with with all or a subset of the
+        dimensions of length 1 removed. This is always `a` itself
+        or a view into `a`.
 
     Examples
     --------
@@ -827,9 +1104,20 @@
     (1, 3, 1)
     >>> np.squeeze(x).shape
     (3,)
+    >>> np.squeeze(x, axis=(2,)).shape
+    (1, 3)
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        squeeze = a.squeeze
+    except AttributeError:
+        return _wrapit(a, 'squeeze')
+    try:
+        # First try to use the new axis= parameter
+        return squeeze(axis=axis)
+    except TypeError:
+        # For backwards compatibility
+        return squeeze()
 
 
 def diagonal(a, offset=0, axis1=0, axis2=1):
@@ -844,6 +1132,27 @@
     removing `axis1` and `axis2` and appending an index to the right equal
     to the size of the resulting diagonals.
 
+    In versions of NumPy prior to 1.7, this function always returned a new,
+    independent array containing a copy of the values in the diagonal.
+
+    In NumPy 1.7 and 1.8, it continues to return a copy of the diagonal,
+    but depending on this fact is deprecated. Writing to the resulting
+    array continues to work as it used to, but a FutureWarning is issued.
+
+    In NumPy 1.9 it returns a read-only view on the original array.
+    Attempting to write to the resulting array will produce an error.
+
+    In NumPy 1.10, it will return a read/write view, Writing to the returned
+    array will alter your original array.
+
+    If you don't write to the array returned by this function, then you can
+    just ignore all of the above.
+
+    If you depend on the current behavior, then we suggest copying the
+    returned array explicitly, i.e., use ``np.diagonal(a).copy()`` instead of
+    just ``np.diagonal(a)``. This will work with both past and future versions
+    of NumPy.
+
     Parameters
     ----------
     a : array_like
@@ -913,7 +1222,7 @@
            [5, 7]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return asarray(a).diagonal(offset, axis1, axis2)
 
 
 def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None):
@@ -972,7 +1281,7 @@
     (2, 3)
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    return asarray(a).trace(offset, axis1, axis2, dtype, out)
 
 def ravel(a, order='C'):
     """
@@ -984,21 +1293,25 @@
     Parameters
     ----------
     a : array_like
-        Input array.  The elements in ``a`` are read in the order specified by
+        Input array.  The elements in `a` are read in the order specified by
         `order`, and packed as a 1-D array.
     order : {'C','F', 'A', 'K'}, optional
-        The elements of ``a`` are read in this order. 'C' means to view
-        the elements in C (row-major) order. 'F' means to view the elements
-        in Fortran (column-major) order. 'A' means to view the elements
-        in 'F' order if a is Fortran contiguous, 'C' order otherwise.
-        'K' means to view the elements in the order they occur in memory,
-        except for reversing the data when strides are negative.
-        By default, 'C' order is used.
+        The elements of `a` are read using this index order. 'C' means to
+        index the elements in C-like order, with the last axis index changing
+        fastest, back to the first axis index changing slowest.   'F' means to
+        index the elements in Fortran-like index order, with the first index
+        changing fastest, and the last index changing slowest. Note that the 'C'
+        and 'F' options take no account of the memory layout of the underlying
+        array, and only refer to the order of axis indexing.  'A' means to read
+        the elements in Fortran-like index order if `a` is Fortran *contiguous*
+        in memory, C-like order otherwise.  'K' means to read the elements in
+        the order they occur in memory, except for reversing the data when
+        strides are negative.  By default, 'C' index order is used.
 
     Returns
     -------
     1d_array : ndarray
-        Output of the same dtype as `a`, and of shape ``(a.size(),)``.
+        Output of the same dtype as `a`, and of shape ``(a.size,)``.
 
     See Also
     --------
@@ -1008,11 +1321,11 @@
 
     Notes
     -----
-    In row-major order, the row index varies the slowest, and the column
-    index the quickest.  This can be generalized to multiple dimensions,
-    where row-major order implies that the index along the first axis
-    varies slowest, and the index along the last quickest.  The opposite holds
-    for Fortran-, or column-major, mode.
+    In C-like (row-major) order, in two dimensions, the row index varies the
+    slowest, and the column index the quickest.  This can be generalized to
+    multiple dimensions, where row-major order implies that the index along the
+    first axis varies slowest, and the index along the last quickest.  The
+    opposite holds for Fortran-like, or column-major, index ordering.
 
     Examples
     --------
@@ -1056,9 +1369,8 @@
     array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11])
 
     """
-    if not hasattr(a, 'ravel'):
-        a = numpypy.array(a)
-    return a.ravel(order=order)
+    return asarray(a).ravel(order)
+
 
 def nonzero(a):
     """
@@ -1180,9 +1492,11 @@
     (2,)
 
     """
-    if not hasattr(a, 'shape'):
-        a = numpypy.array(a)
-    return a.shape
+    try:
+        result = a.shape
+    except AttributeError:
+        result = asarray(a).shape
+    return result
 
 
 def compress(condition, a, axis=None, out=None):
@@ -1217,7 +1531,8 @@
     See Also
     --------
     take, choose, diag, diagonal, select
-    ndarray.compress : Equivalent method.
+    ndarray.compress : Equivalent method in ndarray
+    np.extract: Equivalent method when working on 1-D arrays
     numpy.doc.ufuncs : Section "Output arguments"
 
     Examples
@@ -1244,7 +1559,11 @@
     array([2])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        compress = a.compress
+    except AttributeError:
+        return _wrapit(a, 'compress', condition, axis, out)
+    return compress(condition, axis, out)
 
 
 def clip(a, a_min, a_max, out=None):
@@ -1297,12 +1616,14 @@
     array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8])
 
     """
-    if not hasattr(a, 'clip'):
-        a = numpypy.array(a)
-    return a.clip(a_min, a_max, out=out)
-
-
-def sum(a, axis=None, dtype=None, out=None):
+    try:
+        clip = a.clip
+    except AttributeError:
+        return _wrapit(a, 'clip', a_min, a_max, out)
+    return clip(a_min, a_max, out)
+
+
+def sum(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Sum of array elements over a given axis.
 
@@ -1310,9 +1631,16 @@
     ----------
     a : array_like
         Elements to sum.
-    axis : integer, optional
-        Axis over which the sum is taken. By default `axis` is None,
-        and all elements are summed.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a sum is performed.
+        The default (`axis` = `None`) is perform a sum over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a sum is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     dtype : dtype, optional
         The type of the returned array and of the accumulator in which
         the elements are summed.  By default, the dtype of `a` is used.
@@ -1325,6 +1653,10 @@
         (the shape of `a` with `axis` removed, i.e.,
         ``numpy.delete(a.shape, axis)``).  Its type is preserved. See
         `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1368,13 +1700,25 @@
     -128
 
     """
-    assert dtype is None
-    if not hasattr(a, "sum"):
-        a = numpypy.array(a)
-    return a.sum(axis=axis, out=out)
-
-
-def product (a, axis=None, dtype=None, out=None):
+    if isinstance(a, _gentype):
+        res = _sum_(a)
+        if out is not None:
+            out[...] = res
+            return out
+        return res
+    elif type(a) is not mu.ndarray:
+        try:
+            sum = a.sum
+        except AttributeError:
+            return _methods._sum(a, axis=axis, dtype=dtype,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameters here...
+        return sum(axis=axis, dtype=dtype, out=out)
+    else:
+        return _methods._sum(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
+
+def product (a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Return the product of array elements over a given axis.
 
@@ -1383,10 +1727,10 @@
     prod : equivalent function; see for details.
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def sometrue(a, axis=None, out=None):
+    return um.multiply.reduce(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+
+
+def sometrue(a, axis=None, out=None, keepdims=False):
     """
     Check whether some values are true.
 
@@ -1397,14 +1741,14 @@
     any : equivalent function
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'any'):
-        a = numpypy.array(a)
-    return a.any()
-
-
-def alltrue (a, axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.any(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.any(axis=axis, out=out)
+
+def alltrue (a, axis=None, out=None, keepdims=False):
     """
     Check if all elements of input array are true.
 
@@ -1413,13 +1757,14 @@
     numpy.all : Equivalent function; see for details.
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'all'):
-        a = numpypy.array(a)
-    return a.all()
-
-def any(a,axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.all(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.all(axis=axis, out=out)
+
+def any(a, axis=None, out=None, keepdims=False):
     """
     Test whether any array element along a given axis evaluates to True.
 
@@ -1429,17 +1774,26 @@
     ----------
     a : array_like
         Input array or object that can be converted to an array.
-    axis : int, optional
-        Axis along which a logical OR is performed.  The default
-        (`axis` = `None`) is to perform a logical OR over a flattened
-        input array. `axis` may be negative, in which case it counts
-        from the last to the first axis.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical OR reduction is performed.
+        The default (`axis` = `None`) is perform a logical OR over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a reduction is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     out : ndarray, optional
         Alternate output array in which to place the result.  It must have
         the same shape as the expected output and its type is preserved
         (e.g., if it is of type float, then it will remain so, returning
         1.0 for True and 0.0 for False, regardless of the type of `a`).
         See `doc.ufuncs` (Section "Output arguments") for details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1483,14 +1837,14 @@
     (191614240, 191614240)
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'any'):
-        a = numpypy.array(a)
-    return a.any()
-
-
-def all(a,axis=None, out=None):
+    arr = asanyarray(a)
+
+    try:
+        return arr.any(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.any(axis=axis, out=out)
+
+def all(a, axis=None, out=None, keepdims=False):
     """
     Test whether all array elements along a given axis evaluate to True.
 
@@ -1498,17 +1852,26 @@
     ----------
     a : array_like
         Input array or object that can be converted to an array.
-    axis : int, optional
-        Axis along which a logical AND is performed.
-        The default (`axis` = `None`) is to perform a logical AND
-        over a flattened input array.  `axis` may be negative, in which
-        case it counts from the last to the first axis.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical AND reduction is performed.
+        The default (`axis` = `None`) is perform a logical OR over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a reduction is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     out : ndarray, optional
         Alternate output array in which to place the result.
         It must have the same shape as the expected output and its
         type is preserved (e.g., if ``dtype(out)`` is float, the result
         will consist of 0.0's and 1.0's).  See `doc.ufuncs` (Section
         "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1547,12 +1910,12 @@
     (28293632, 28293632, array([ True], dtype=bool))
 
     """
-    assert axis is None
-    assert out is None
-    if not hasattr(a, 'all'):
-        a = numpypy.array(a)
-    return a.all()
-
+    arr = asanyarray(a)
+
+    try:
+        return arr.all(axis=axis, out=out, keepdims=keepdims)
+    except TypeError:
+        return arr.all(axis=axis, out=out)
 
 def cumsum (a, axis=None, dtype=None, out=None):
     """
@@ -1592,6 +1955,8 @@
 
     trapz : Integration of array values using the composite trapezoidal rule.
 
+    diff :  Calculate the n-th order discrete difference along given axis.
+
     Notes
     -----
     Arithmetic is modular when using integer types, and no error is
@@ -1616,7 +1981,11 @@
            [ 4,  9, 15]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumsum = a.cumsum
+    except AttributeError:
+        return _wrapit(a, 'cumsum', axis, dtype, out)
+    return cumsum(axis, dtype, out)
 
 
 def cumproduct(a, axis=None, dtype=None, out=None):
@@ -1629,7 +1998,11 @@
     cumprod : equivalent function; see for details.
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumprod = a.cumprod
+    except AttributeError:
+        return _wrapit(a, 'cumprod', axis, dtype, out)
+    return cumprod(axis, dtype, out)
 
 
 def ptp(a, axis=None, out=None):
@@ -1670,10 +2043,14 @@
     array([1, 1])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def amax(a, axis=None, out=None):
+    try:
+        ptp = a.ptp
+    except AttributeError:
+        return _wrapit(a, 'ptp', axis, out)
+    return ptp(axis, out)
+
+
+def amax(a, axis=None, out=None, keepdims=False):
     """
     Return the maximum of an array or maximum along an axis.
 
@@ -1682,11 +2059,15 @@
     a : array_like
         Input data.
     axis : int, optional
-        Axis along which to operate.  By default flattened input is used.
+        Axis along which to operate.  By default, flattened input is used.
     out : ndarray, optional
-        Alternate output array in which to place the result.  Must be of
-        the same shape and buffer length as the expected output.  See
-        `doc.ufuncs` (Section "Output arguments") for more details.
+        Alternative output array in which to place the result.  Must
+        be of the same shape and buffer length as the expected output.
+        See `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1697,27 +2078,40 @@
 
     See Also
     --------
-    nanmax : NaN values are ignored instead of being propagated.
-    fmax : same behavior as the C99 fmax function.
-    argmax : indices of the maximum values.
+    amin :
+        The minimum value of an array along a given axis, propagating any NaNs.
+    nanmax :
+        The maximum value of an array along a given axis, ignoring any NaNs.
+    maximum :
+        Element-wise maximum of two arrays, propagating any NaNs.
+    fmax :
+        Element-wise maximum of two arrays, ignoring any NaNs.
+    argmax :
+        Return the indices of the maximum values.
+
+    nanmin, minimum, fmin
 
     Notes
     -----
     NaN values are propagated, that is if at least one item is NaN, the
-    corresponding max value will be NaN as well.  To ignore NaN values
+    corresponding max value will be NaN as well. To ignore NaN values
     (MATLAB behavior), please use nanmax.
 
+    Don't use `amax` for element-wise comparison of 2 arrays; when
+    ``a.shape[0]`` is 2, ``maximum(a[0], a[1])`` is faster than
+    ``amax(a, axis=0)``.
+
     Examples
     --------
     >>> a = np.arange(4).reshape((2,2))
     >>> a
     array([[0, 1],
            [2, 3]])
-    >>> np.amax(a)
+    >>> np.amax(a)           # Maximum of the flattened array
     3
-    >>> np.amax(a, axis=0)
+    >>> np.amax(a, axis=0)   # Maxima along the first axis
     array([2, 3])
-    >>> np.amax(a, axis=1)
+    >>> np.amax(a, axis=1)   # Maxima along the second axis
     array([1, 3])
 
     >>> b = np.arange(5, dtype=np.float)
@@ -1728,14 +2122,19 @@
     4.0
 
     """
-    if not hasattr(a, "max"):
-        a = numpypy.array(a)
-    if a.size < 1:
-        return numpypy.array([])
-    return a.max(axis=axis, out=out)
-
-
-def amin(a, axis=None, out=None):
+    if type(a) is not mu.ndarray:
+        try:
+            amax = a.max
+        except AttributeError:
+            return _methods._amax(a, axis=axis,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameter
+        return amax(axis=axis, out=out)
+    else:
+        return _methods._amax(a, axis=axis,
+                            out=out, keepdims=keepdims)
+
+def amin(a, axis=None, out=None, keepdims=False):
     """
     Return the minimum of an array or minimum along an axis.
 
@@ -1744,30 +2143,47 @@
     a : array_like
         Input data.
     axis : int, optional
-        Axis along which to operate.  By default a flattened input is used.
+        Axis along which to operate.  By default, flattened input is used.
     out : ndarray, optional
         Alternative output array in which to place the result.  Must
         be of the same shape and buffer length as the expected output.
         See `doc.ufuncs` (Section "Output arguments") for more details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
-    amin : ndarray
-        A new array or a scalar array with the result.
+    amin : ndarray or scalar
+        Minimum of `a`. If `axis` is None, the result is a scalar value.
+        If `axis` is given, the result is an array of dimension
+        ``a.ndim - 1``.
 
     See Also
     --------
-    nanmin: nan values are ignored instead of being propagated
-    fmin: same behavior as the C99 fmin function
-    argmin: Return the indices of the minimum values.
-
-    amax, nanmax, fmax
+    amax :
+        The maximum value of an array along a given axis, propagating any NaNs.
+    nanmin :
+        The minimum value of an array along a given axis, ignoring any NaNs.
+    minimum :
+        Element-wise minimum of two arrays, propagating any NaNs.
+    fmin :
+        Element-wise minimum of two arrays, ignoring any NaNs.
+    argmin :
+        Return the indices of the minimum values.
+
+    nanmax, maximum, fmax
 
     Notes
     -----
-    NaN values are propagated, that is if at least one item is nan, the
-    corresponding min value will be nan as well. To ignore NaN values (matlab
-    behavior), please use nanmin.
+    NaN values are propagated, that is if at least one item is NaN, the
+    corresponding min value will be NaN as well. To ignore NaN values
+    (MATLAB behavior), please use nanmin.
+
+    Don't use `amin` for element-wise comparison of 2 arrays; when
+    ``a.shape[0]`` is 2, ``minimum(a[0], a[1])`` is faster than
+    ``amin(a, axis=0)``.
 
     Examples
     --------
@@ -1777,9 +2193,9 @@
            [2, 3]])
     >>> np.amin(a)           # Minimum of the flattened array
     0
-    >>> np.amin(a, axis=0)         # Minima along the first axis
+    >>> np.amin(a, axis=0)   # Minima along the first axis
     array([0, 1])
-    >>> np.amin(a, axis=1)         # Minima along the second axis
+    >>> np.amin(a, axis=1)   # Minima along the second axis
     array([0, 2])
 
     >>> b = np.arange(5, dtype=np.float)
@@ -1790,11 +2206,17 @@
     0.0
 
     """
-    if not hasattr(a, 'min'):
-        a = numpypy.array(a)
-    if a.size < 1:
-        return numpypy.array([])
-    return a.min(axis=axis, out=out)
+    if type(a) is not mu.ndarray:
+        try:
+            amin = a.min
+        except AttributeError:
+            return _methods._amin(a, axis=axis,
+                                out=out, keepdims=keepdims)
+        # NOTE: Dropping the keepdims parameter
+        return amin(axis=axis, out=out)
+    else:
+        return _methods._amin(a, axis=axis,
+                            out=out, keepdims=keepdims)
 
 def alen(a):
     """
@@ -1807,7 +2229,7 @@
 
     Returns
     -------
-    l : int
+    alen : int
        Length of the first dimension of `a`.
 
     See Also
@@ -1823,12 +2245,13 @@
     7
 
     """
-    if not hasattr(a, 'shape'):
-        a = numpypy.array(a)
-    return a.shape[0]
-
-
-def prod(a, axis=None, dtype=None, out=None):
+    try:
+        return len(a)
+    except TypeError:
+        return len(array(a, ndmin=1))
+
+
+def prod(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Return the product of array elements over a given axis.
 
@@ -1836,9 +2259,16 @@
     ----------
     a : array_like
         Input data.
-    axis : int, optional
-        Axis over which the product is taken.  By default, the product
-        of all elements is calculated.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a product is performed.
+        The default (`axis` = `None`) is perform a product over all
+        the dimensions of the input array. `axis` may be negative, in
+        which case it counts from the last to the first axis.
+
+        .. versionadded:: 1.7.0
+
+        If this is a tuple of ints, a product is performed on multiple
+        axes, instead of a single axis or all the axes as before.
     dtype : data-type, optional
         The data-type of the returned array, as well as of the accumulator
         in which the elements are multiplied.  By default, if `a` is of
@@ -1849,6 +2279,10 @@
         Alternative output array in which to place the result. It must have
         the same shape as the expected output, but the type of the
         output values will be cast if necessary.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -1902,8 +2336,16 @@
     True
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
+    if type(a) is not mu.ndarray:
+        try:
+            prod = a.prod
+        except AttributeError:
+            return _methods._prod(a, axis=axis, dtype=dtype,
+                                out=out, keepdims=keepdims)
+        return prod(axis=axis, dtype=dtype, out=out)
+    else:
+        return _methods._prod(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
 
 def cumprod(a, axis=None, dtype=None, out=None):
     """
@@ -1965,7 +2407,11 @@
            [  4,  20, 120]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        cumprod = a.cumprod
+    except AttributeError:
+        return _wrapit(a, 'cumprod', axis, dtype, out)
+    return cumprod(axis, dtype, out)
 
 
 def ndim(a):
@@ -1999,9 +2445,10 @@
     0
 
     """
-    if not hasattr(a, 'ndim'):
-        a = numpypy.array(a)
-    return a.ndim
+    try:
+        return a.ndim
+    except AttributeError:
+        return asarray(a).ndim
 
 
 def rank(a):
@@ -2044,9 +2491,10 @@
     0
 
     """
-    if not hasattr(a, 'ndim'):
-        a = numpypy.array(a)
-    return a.ndim
+    try:
+        return a.ndim
+    except AttributeError:
+        return asarray(a).ndim
 
 
 def size(a, axis=None):
@@ -2083,7 +2531,16 @@
     2
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    if axis is None:
+        try:
+            return a.size
+        except AttributeError:
+            return asarray(a).size
+    else:
+        try:
+            return a.shape[axis]
+        except AttributeError:
+            return asarray(a).shape[axis]
 
 
 def around(a, decimals=0, out=None):
@@ -2152,7 +2609,11 @@
     array([ 0,  0,  0, 10])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    try:
+        round = a.round
+    except AttributeError:
+        return _wrapit(a, 'round', decimals, out)
+    return round(decimals, out)
 
 
 def round_(a, decimals=0, out=None):
@@ -2166,10 +2627,14 @@
     around : equivalent function
 
     """
-    raise NotImplementedError('Waiting on interp level method')
-
-
-def mean(a, axis=None, dtype=None, out=None):
+    try:
+        round = a.round
+    except AttributeError:
+        return _wrapit(a, 'round', decimals, out)
+    return round(decimals, out)
+
+
+def mean(a, axis=None, dtype=None, out=None, keepdims=False):
     """
     Compute the arithmetic mean along the specified axis.
 
@@ -2194,6 +2659,10 @@
         is ``None``; if provided, it must have the same shape as the
         expected output, but the type will be cast if necessary.
         See `doc.ufuncs` for details.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2204,6 +2673,7 @@
     See Also
     --------
     average : Weighted average
+    std, var, nanmean, nanstd, nanvar
 
     Notes
     -----
@@ -2240,14 +2710,17 @@
     0.55000000074505806
 
     """
-    assert dtype is None
-    assert out is None
-    if not hasattr(a, "mean"):
-        a = numpypy.array(a)
-    return a.mean(axis=axis)
-
-
-def std(a, axis=None, dtype=None, out=None, ddof=0):
+    if type(a) is not mu.ndarray:
+        try:
+            mean = a.mean
+            return mean(axis=axis, dtype=dtype, out=out)
+        except AttributeError:
+            pass
+
+    return _methods._mean(a, axis=axis, dtype=dtype,
+                            out=out, keepdims=keepdims)
+
+def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
     """
     Compute the standard deviation along the specified axis.
 
@@ -2274,6 +2747,10 @@
         Means Delta Degrees of Freedom.  The divisor used in calculations
         is ``N - ddof``, where ``N`` represents the number of elements.
         By default `ddof` is zero.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2283,7 +2760,7 @@
 
     See Also
     --------
-    var, mean
+    var, mean, nanmean, nanstd, nanvar
     numpy.doc.ufuncs : Section "Output arguments"
 
     Notes
@@ -2291,14 +2768,15 @@
     The standard deviation is the square root of the average of the squared
     deviations from the mean, i.e., ``std = sqrt(mean(abs(x - x.mean())**2))``.
 
-    The average squared deviation is normally calculated as ``x.sum() / N``, where
-    ``N = len(x)``.  If, however, `ddof` is specified, the divisor ``N - ddof``
-    is used instead. In standard statistical practice, ``ddof=1`` provides an
-    unbiased estimator of the variance of the infinite population. ``ddof=0``
-    provides a maximum likelihood estimate of the variance for normally
-    distributed variables. The standard deviation computed in this function
-    is the square root of the estimated variance, so even with ``ddof=1``, it
-    will not be an unbiased estimate of the standard deviation per se.
+    The average squared deviation is normally calculated as
+    ``x.sum() / N``, where ``N = len(x)``.  If, however, `ddof` is specified,
+    the divisor ``N - ddof`` is used instead. In standard statistical
+    practice, ``ddof=1`` provides an unbiased estimator of the variance
+    of the infinite population. ``ddof=0`` provides a maximum likelihood
+    estimate of the variance for normally distributed variables. The
+    standard deviation computed in this function is the square root of
+    the estimated variance, so even with ``ddof=1``, it will not be an
+    unbiased estimate of the standard deviation per se.
 
     Note that, for complex numbers, `std` takes the absolute
     value before squaring, so that the result is always real and nonnegative.
@@ -2333,15 +2811,18 @@
     0.44999999925552653
 
     """
-    assert dtype is None
-    assert out is None
-    assert ddof == 0
-    if not hasattr(a, "std"):
-        a = numpypy.array(a)
-    return a.std(axis=axis)
-
-
-def var(a, axis=None, dtype=None, out=None, ddof=0):
+    if type(a) is not mu.ndarray:
+        try:
+            std = a.std
+            return std(axis=axis, dtype=dtype, out=out, ddof=ddof)
+        except AttributeError:
+            pass
+
+    return _methods._std(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
+                                keepdims=keepdims)
+
+def var(a, axis=None, dtype=None, out=None, ddof=0,
+                            keepdims=False):
     """
     Compute the variance along the specified axis.
 
@@ -2369,6 +2850,10 @@
         "Delta Degrees of Freedom": the divisor used in the calculation is
         ``N - ddof``, where ``N`` represents the number of elements. By
         default `ddof` is zero.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
 
     Returns
     -------
@@ -2378,8 +2863,7 @@
 
     See Also
     --------
-    std : Standard deviation
-    mean : Average
+    std , mean, nanmean, nanstd, nanvar
     numpy.doc.ufuncs : Section "Output arguments"
 
     Notes
@@ -2408,9 +2892,9 @@
     >>> a = np.array([[1,2],[3,4]])
     >>> np.var(a)
     1.25
-    >>> np.var(a,0)
+    >>> np.var(a, axis=0)
     array([ 1.,  1.])
-    >>> np.var(a,1)
+    >>> np.var(a, axis=1)
     array([ 0.25,  0.25])
 
     In single precision, var() can be inaccurate:
@@ -2421,7 +2905,7 @@
     >>> np.var(a)
     0.20405951142311096
 
-    Computing the standard deviation in float64 is more accurate:
+    Computing the variance in float64 is more accurate:
 
     >>> np.var(a, dtype=np.float64)
     0.20249999932997387
@@ -2429,9 +2913,12 @@
     0.20250000000000001
 
     """
-    assert dtype is None
-    assert out is None
-    assert ddof == 0
-    if not hasattr(a, "var"):
-        a = numpypy.array(a)
-    return a.var(axis=axis)
+    if type(a) is not mu.ndarray:
+        try:
+            var = a.var
+            return var(axis=axis, dtype=dtype, out=out, ddof=ddof)
+        except AttributeError:
+            pass
+
+    return _methods._var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
+                                keepdims=keepdims)
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -1,21 +1,24 @@
+from __future__ import division, absolute_import, print_function
+
 __all__ = [
-           'newaxis', 'ufunc',
+           'newaxis', 'ufunc', 'argwhere',
            'asarray', 'asanyarray', 'base_repr',
            'array_repr', 'array_str', 'set_string_function',
-           'array_equal', 'outer', 'vdot', 'identity', 'little_endian',
-           'Inf', 'inf', 'infty', 'Infinity', 'nan', 'NaN', 'False_', 'True_',
-           'seterr',
+           'array_equal', 'array_equiv', 'outer', 'vdot', 'identity', 'little_endian',
+           'seterr', 'flatnonzero',
+           'Inf', 'inf', 'infty', 'Infinity',
+           'nan', 'NaN', 'False_', 'True_',
           ]
 
 import sys
-import multiarray
-from multiarray import *
+from . import multiarray
+from .multiarray import *
 del set_string_function
 del typeinfo
-import umath
-from umath import *
-import numerictypes
-from numerictypes import *
+from . import umath
+from .umath import *
+from . import numerictypes
+from .numerictypes import *
 
 def extend_all(module):
     adict = {}
@@ -41,6 +44,76 @@
 def seterr(**args):
     return args
 
+def asarray(a, dtype=None, order=None):
+    """
+    Convert the input to an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data, in any form that can be converted to an array.  This
+        includes lists, lists of tuples, tuples, tuples of tuples, tuples
+        of lists and ndarrays.
+    dtype : data-type, optional
+        By default, the data-type is inferred from the input data.
+    order : {'C', 'F'}, optional
+        Whether to use row-major ('C') or column-major ('F' for FORTRAN)
+        memory representation.  Defaults to 'C'.
+
+    Returns
+    -------
+    out : ndarray
+        Array interpretation of `a`.  No copy is performed if the input
+        is already an ndarray.  If `a` is a subclass of ndarray, a base
+        class ndarray is returned.
+
+    See Also
+    --------
+    asanyarray : Similar function which passes through subclasses.
+    ascontiguousarray : Convert input to a contiguous array.
+    asfarray : Convert input to a floating point ndarray.
+    asfortranarray : Convert input to an ndarray with column-major
+                     memory order.
+    asarray_chkfinite : Similar function which checks input for NaNs and Infs.
+    fromiter : Create an array from an iterator.
+    fromfunction : Construct an array by executing a function on grid
+                   positions.
+
+    Examples
+    --------
+    Convert a list into an array:
+
+    >>> a = [1, 2]
+    >>> np.asarray(a)
+    array([1, 2])
+
+    Existing arrays are not copied:
+
+    >>> a = np.array([1, 2])
+    >>> np.asarray(a) is a
+    True
+
+    If `dtype` is set, array is copied only if dtype does not match:
+
+    >>> a = np.array([1, 2], dtype=np.float32)
+    >>> np.asarray(a, dtype=np.float32) is a
+    True
+    >>> np.asarray(a, dtype=np.float64) is a
+    False
+
+    Contrary to `asanyarray`, ndarray subclasses are not passed through:
+
+    >>> issubclass(np.matrix, np.ndarray)
+    True
+    >>> a = np.matrix([[1, 2]])
+    >>> np.asarray(a) is a
+    False
+    >>> np.asanyarray(a) is a
+    True
+
+    """
+    return array(a, dtype, copy=False, order=order)
+
 def asanyarray(a, dtype=None, order=None):
     """
     Convert the input to an ndarray, but pass ndarray subclasses through.
@@ -93,6 +166,85 @@
     """
     return array(a, dtype, copy=False, order=order, subok=True)
 
+def argwhere(a):
+    """
+    Find the indices of array elements that are non-zero, grouped by element.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+
+    Returns
+    -------
+    index_array : ndarray
+        Indices of elements that are non-zero. Indices are grouped by element.
+
+    See Also
+    --------
+    where, nonzero
+
+    Notes
+    -----
+    ``np.argwhere(a)`` is the same as ``np.transpose(np.nonzero(a))``.
+
+    The output of ``argwhere`` is not suitable for indexing arrays.
+    For this purpose use ``where(a)`` instead.
+
+    Examples
+    --------
+    >>> x = np.arange(6).reshape(2,3)
+    >>> x
+    array([[0, 1, 2],
+           [3, 4, 5]])
+    >>> np.argwhere(x>1)
+    array([[0, 2],
+           [1, 0],
+           [1, 1],
+           [1, 2]])
+
+    """
+    return transpose(asanyarray(a).nonzero())
+
+def flatnonzero(a):
+    """
+    Return indices that are non-zero in the flattened version of a.
+
+    This is equivalent to a.ravel().nonzero()[0].
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array.
+
+    Returns
+    -------
+    res : ndarray
+        Output array, containing the indices of the elements of `a.ravel()`
+        that are non-zero.
+
+    See Also
+    --------
+    nonzero : Return the indices of the non-zero elements of the input array.
+    ravel : Return a 1-D array containing the elements of the input array.
+
+    Examples
+    --------
+    >>> x = np.arange(-2, 3)
+    >>> x
+    array([-2, -1,  0,  1,  2])
+    >>> np.flatnonzero(x)
+    array([0, 1, 3, 4])
+
+    Use the indices of the non-zero elements as an index array to extract
+    these elements:
+
+    >>> x.ravel()[np.flatnonzero(x)]
+    array([-2, -1,  1,  2])
+
+    """
+    return a.ravel().nonzero()[0]
+
 def base_repr(number, base=2, padding=0):
     """
     Return a string representation of a number in the given base system.
@@ -148,7 +300,7 @@
 
 
 #Use numarray's printing function
-from arrayprint import array2string
+from .arrayprint import array2string
 
 _typelessdata = [int_, float_]#, complex_]
 # XXX
@@ -381,75 +533,49 @@
         return False
     return bool((a1 == a2).all())
 
-def asarray(a, dtype=None, order=None):
+def array_equiv(a1, a2):
     """
-    Convert the input to an array.
+    Returns True if input arrays are shape consistent and all elements equal.
+
+    Shape consistent means they are either the same shape, or one input array
+    can be broadcasted to create the same shape as the other one.
 
     Parameters
     ----------
-    a : array_like
-        Input data, in any form that can be converted to an array.  This
-        includes lists, lists of tuples, tuples, tuples of tuples, tuples
-        of lists and ndarrays.
-    dtype : data-type, optional
-        By default, the data-type is inferred from the input data.
-    order : {'C', 'F'}, optional
-        Whether to use row-major ('C') or column-major ('F' for FORTRAN)
-        memory representation.  Defaults to 'C'.
+    a1, a2 : array_like
+        Input arrays.
 
     Returns
     -------
-    out : ndarray
-        Array interpretation of `a`.  No copy is performed if the input
-        is already an ndarray.  If `a` is a subclass of ndarray, a base
-        class ndarray is returned.
-
-    See Also
-    --------
-    asanyarray : Similar function which passes through subclasses.
-    ascontiguousarray : Convert input to a contiguous array.
-    asfarray : Convert input to a floating point ndarray.
-    asfortranarray : Convert input to an ndarray with column-major
-                     memory order.
-    asarray_chkfinite : Similar function which checks input for NaNs and Infs.
-    fromiter : Create an array from an iterator.
-    fromfunction : Construct an array by executing a function on grid
-                   positions.
+    out : bool
+        True if equivalent, False otherwise.
 
     Examples
     --------
-    Convert a list into an array:
-

From noreply at buildbot.pypy.org  Tue Oct 22 20:46:41 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 22 Oct 2013 20:46:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Don't reset virtuals on call_assembler
Message-ID: <20131022184641.1D25A1C0144@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67520:b741f795474c
Date: 2013-10-22 20:45 +0200
http://bitbucket.org/pypy/pypy/changeset/b741f795474c/

Log:	Don't reset virtuals on call_assembler

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -6,7 +6,7 @@
     def __init__(self):
         self.reset()
 
-    def reset(self):
+    def reset(self, reset_virtuals=True):
         # contains boxes where the class is already known
         self.known_class_boxes = {}
         # store the boxes that contain newly allocated objects, this maps the
@@ -14,7 +14,8 @@
         # escaped the trace or not (True means the box never escaped, False
         # means it did escape), its presences in the mapping shows that it was
         # allocated inside the trace
-        self.new_boxes = {}
+        if reset_virtuals:
+            self.new_boxes = {}
         # Tracks which boxes should be marked as escaped when the key box
         # escapes.
         self.dependencies = {}
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2058,7 +2058,7 @@
                 duplicates[box] = None
 
     def reached_loop_header(self, greenboxes, redboxes, resumedescr):
-        self.heapcache.reset()
+        self.heapcache.reset(reset_virtuals=False)
 
         duplicates = {}
         self.remove_consts_and_duplicates(redboxes, len(redboxes),
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3359,6 +3359,26 @@
         assert res == main(1)
         self.check_resops(call=0, getfield_gc=0)
 
+    def test_isvirtual_call_assembler(self):
+        driver = JitDriver(greens = ['code'], reds = ['n'])
+
+        @look_inside_iff(lambda t1, t2: isvirtual(t1))
+        def g(t1, t2):
+            return t1[0] == t2[0]
+
+        def f(code, n):
+            while n > 0:
+                driver.can_enter_jit(code=code, n=n)
+                driver.jit_merge_point(code=code, n=n)
+                t = (1, 2, n)
+                if code:
+                    f(0, 3)
+                g(t, (1, 2, n))
+                n -= 1
+
+        self.meta_interp(f, [1, 10], inline=True)
+        self.check_resops(call=0, call_may_force=0, call_assembler=2)
+
     def test_reuse_elidable_result(self):
         driver = JitDriver(reds=['n', 's'], greens = [])
         def main(n):

From noreply at buildbot.pypy.org  Tue Oct 22 20:46:42 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 22 Oct 2013 20:46:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: Improve pypyjit.py enough to make it
	work
Message-ID: <20131022184642.6F0DB1C0144@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67521:ff807f024c0a
Date: 2013-10-22 20:45 +0200
http://bitbucket.org/pypy/pypy/changeset/ff807f024c0a/

Log:	Improve pypyjit.py enough to make it work

diff --git a/pypy/tool/pypyjit.py b/pypy/tool/pypyjit.py
--- a/pypy/tool/pypyjit.py
+++ b/pypy/tool/pypyjit.py
@@ -28,15 +28,16 @@
 config.translation.backendopt.inline_threshold = 0.1
 config.translation.gc = 'boehm'
 config.translating = True
+config.translation.rweakref = False
 set_opt_level(config, level='jit')
 config.objspace.allworkingmodules = False
 config.objspace.usemodules.pypyjit = True
 config.objspace.usemodules.array = False
-config.objspace.usemodules._weakref = True
+config.objspace.usemodules._weakref = False
 config.objspace.usemodules._sre = False
 config.objspace.usemodules._lsprof = False
 #
-config.objspace.usemodules._ffi = True
+config.objspace.usemodules._ffi = False
 config.objspace.usemodules.micronumpy = False
 #
 set_pypy_opt_level(config, level='jit')
@@ -101,7 +102,7 @@
 
     from rpython.jit.codewriter.codewriter import CodeWriter
     CodeWriter.debug = True
-    from rpython.jit.tl.pypyjit_child import run_child
+    from pypy.tool.pypyjit_child import run_child
     run_child(globals(), locals())
 
 
diff --git a/pypy/tool/pypyjit_demo.py b/pypy/tool/pypyjit_demo.py
--- a/pypy/tool/pypyjit_demo.py
+++ b/pypy/tool/pypyjit_demo.py
@@ -1,27 +1,20 @@
-import pypyjit
-pypyjit.set_param(threshold=200)
 
-kwargs = {"z": 1}
+def g(i):
+    k = 0
+    while k < 3:
+        k += 1
+    return i + 1
 
-def f(*args, **kwargs):
-    result = g(1, *args, **kwargs)
-    return result + 2
+def f(x):
+    for i in range(10000):
+        t = (1, 2, i)
+        i = g(i)
+        x == t
 
-def g(x, y, z=2):
-    return x - y + z
-
-def main():
-    res = 0
-    i = 0
-    while i < 10000:
-        res = f(res, z=i)
-        g(1, res, **kwargs)
-        i += 1
-    return res
 
 
 try:
-    print main()
+    f((1, 2, 3))
 
 except Exception, e:
     print "Exception: ", type(e)

From noreply at buildbot.pypy.org  Wed Oct 23 06:44:17 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 23 Oct 2013 06:44:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test_rctime on osx
Message-ID: <20131023044417.5B00F1C0163@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67522:d62e21894538
Date: 2013-10-22 21:43 -0700
http://bitbucket.org/pypy/pypy/changeset/d62e21894538/

Log:	fix test_rctime on osx

diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -139,6 +139,10 @@
             assert rctime.ctime(t) != rctime.asctime(rctime.gmtime(t))
         ltime = rctime.localtime()
         assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
+        try:
+            assert rctime.asctime((12345,) + (0,) * 8).split()[-1] == '12345'
+        except ValueError:
+            pass  # some OS (ie POSIXes besides Linux) reject year > 9999
 
     def test_accept2dyear_access(self):
         import time as rctime
@@ -146,7 +150,8 @@
         accept2dyear = rctime.accept2dyear
         del rctime.accept2dyear
         try:
-            assert rctime.asctime((12345,) + (0,) * 8).split()[-1] == '12345'
+            # with year >= 1900 this shouldn't need to access accept2dyear
+            assert rctime.asctime((2000,) + (0,) * 8).split()[-1] == '2000'
         finally:
             rctime.accept2dyear = accept2dyear
 

From noreply at buildbot.pypy.org  Wed Oct 23 11:30:02 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 23 Oct 2013 11:30:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: document a branch
Message-ID: <20131023093002.624791C3043@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67523:3738e0241da2
Date: 2013-10-23 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/3738e0241da2/

Log:	document a branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -111,3 +111,6 @@
 
 .. branch: incremental-gc
 Added the new incminimark GC which performs GC in incremental steps
+
+.. branch: fast_cffi_list_init
+fastpath for cffi.new("long[]")

From noreply at buildbot.pypy.org  Wed Oct 23 11:46:15 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 23 Oct 2013 11:46:15 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: Change the list
 comprehension optimization to not depend on
Message-ID: <20131023094615.6C5CF1C0204@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67524:6535af3b2988
Date: 2013-10-23 06:52 +0200
http://bitbucket.org/pypy/pypy/changeset/6535af3b2988/

Log:	Change the list comprehension optimization to not depend on
	lltypesystem/rlist.py any more (in particular, we want here to
	remove the fact that fixed-size and var-size lists share the GcArray
	part).

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -303,20 +303,6 @@
         return s_Bool
     op_contains.can_only_throw = []
 
-    def hint(lst, *args_s):
-        hints = args_s[-1].const
-        if 'maxlength' in hints:
-            # only for iteration over lists or dicts at the moment,
-            # not over an iterator object (because it has no known length)
-            s_iterable = args_s[0]
-            if isinstance(s_iterable, (SomeList, SomeDict)):
-                lst = SomeList(lst.listdef) # create a fresh copy
-                lst.listdef.resize()
-                lst.listdef.listitem.hint_maxlength = True
-        elif 'fence' in hints:
-            lst = lst.listdef.offspring()
-        return lst
-
     def getslice(lst, s_start, s_stop):
         check_negative_slice(s_start, s_stop)
         return lst.listdef.offspring()
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -375,6 +375,47 @@
         hop.exception_is_here()
         hop.gendirectcall(r_list.LIST._ll_resize_hint, v_list, v_sizehint)
 
+def newlist_fixed(length):
+    """Create a new fixed-size list of the given length.  The elements
+    are initialized with 0/None/whatever."""
+    return [None] * length    # xxx when not translated, assume list of objects
+
+class Entry(ExtRegistryEntry):
+    _about_ = newlist_fixed
+
+    def compute_result_annotation(self, s_length):
+        from rpython.annotator import model as annmodel
+        assert isinstance(s_length, annmodel.SomeInteger)
+        return self.bookkeeper.newlist()
+
+    def specialize_call(self, hop):
+        from rpython.rtyper.lltypesystem import lltype
+        from rpython.rtyper.rlist import ll_newlist_fixed
+        r_list = hop.r_result
+        v_count, = hop.inputargs(lltype.Signed)
+        cLIST = hop.inputconst(lltype.Void, r_list.LIST)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_newlist_fixed, cLIST, v_count)
+
+def _is_list_or_dict(x):
+    return isinstance(x, (list, dict))
+
+class Entry(ExtRegistryEntry):
+    _about_ = _is_list_or_dict
+
+    def compute_result_annotation(self, s_x):
+        from rpython.annotator import model as annmodel
+        if annmodel.s_None.contains(s_x):
+            return annmodel.s_ImpossibleValue
+        s = annmodel.SomeBool()
+        s.const = (isinstance(s_x, annmodel.SomeList) or
+                   isinstance(s_x, annmodel.SomeDict))
+        return s
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        return hop.inputconst(hop.r_result.lowleveltype, hop.s_result.const)
+
 # ____________________________________________________________
 #
 # id-like functions.  The idea is that calling hash() or id() is not
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -549,6 +549,27 @@
     r = interpret(f, [29])
     assert r == 1
 
+def test_newlist_fixed():
+    def f(i):
+        l = newlist_fixed(i)
+        return len(l)
+
+    r = interpret(f, [5])
+    assert r == 5
+
+def test_is_list_or_dict():
+    from rpython.rlib.objectmodel import _is_list_or_dict
+    #
+    def f(x):
+        return _is_list_or_dict(x)
+    r = interpret(f, [5])
+    assert r is False
+    #
+    def f(x):
+        return _is_list_or_dict([x])
+    r = interpret(f, [5])
+    assert r is True
+
 def test_import_from_mixin():
     class M:    # old-style
         def f(self): pass
diff --git a/rpython/rtyper/lltypesystem/rlist.py b/rpython/rtyper/lltypesystem/rlist.py
--- a/rpython/rtyper/lltypesystem/rlist.py
+++ b/rpython/rtyper/lltypesystem/rlist.py
@@ -116,34 +116,6 @@
         result.items = malloc(self.LIST.items.TO, n)
         return result
 
-    def rtype_method_append(self, hop):
-        if getattr(self.listitem, 'hint_maxlength', False):
-            v_lst, v_value = hop.inputargs(self, self.item_repr)
-            hop.exception_cannot_occur()
-            hop.gendirectcall(ll_append_noresize, v_lst, v_value)
-        else:
-            AbstractListRepr.rtype_method_append(self, hop)
-
-    def rtype_hint(self, hop):
-        optimized = getattr(self.listitem, 'hint_maxlength', False)
-        hints = hop.args_s[-1].const
-        if 'maxlength' in hints:
-            if optimized:
-                v_list = hop.inputarg(self, arg=0)
-                v_maxlength = self._get_v_maxlength(hop)
-                hop.llops.gendirectcall(ll_set_maxlength, v_list, v_maxlength)
-                return v_list
-        if 'fence' in hints:
-            v_list = hop.inputarg(self, arg=0)
-            if isinstance(hop.r_result, FixedSizeListRepr):
-                if optimized and 'exactlength' in hints:
-                    llfn = ll_list2fixed_exact
-                else:
-                    llfn = ll_list2fixed
-                v_list = hop.llops.gendirectcall(llfn, v_list)
-            return v_list
-        return AbstractListRepr.rtype_hint(self, hop)
-
 
 class FixedSizeListRepr(AbstractFixedSizeListRepr, BaseListRepr):
 
@@ -401,26 +373,6 @@
         llops.gendirectcall(ll_setitem_nonneg, v_func, v_result, ci, v_item)
     return v_result
 
-# special operations for list comprehension optimization
-def ll_set_maxlength(l, n):
-    LIST = typeOf(l).TO
-    l.items = malloc(LIST.items.TO, n)
-
-def ll_list2fixed(l):
-    n = l.length
-    olditems = l.items
-    if n == len(olditems):
-        return olditems
-    else:
-        LIST = typeOf(l).TO
-        newitems = malloc(LIST.items.TO, n)
-        rgc.ll_arraycopy(olditems, newitems, 0, 0, n)
-        return newitems
-
-def ll_list2fixed_exact(l):
-    ll_assert(l.length == len(l.items), "ll_list2fixed_exact: bad length")
-    return l.items
-
 # ____________________________________________________________
 #
 #  Iteration.
diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py
--- a/rpython/rtyper/rlist.py
+++ b/rpython/rtyper/rlist.py
@@ -489,6 +489,11 @@
             i += 1
     return l
 
+ at jit.oopspec("newlist(count)")
+def ll_newlist_fixed(LIST, count):
+    # called by rtyping of objectmodel.newlist_fixed()
+    return LIST.ll_newlist(count)
+
 
 # return a nullptr() if lst is a list of pointers it, else None.
 def ll_null_item(lst):
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -617,16 +617,18 @@
 # ____________________________________________________________
 
 def detect_list_comprehension(graph):
-    """Look for the pattern:            Replace it with marker operations:
+    """Look for the pattern:            Replace it with these operations:
 
-                                         v0 = newlist()
-        v2 = newlist()                   v1 = hint(v0, iterable, {'maxlength'})
+        v2 = newlist()                   v1 = simple_call(RListCompr)
+        ...                              ...
+        v4 = iter(v3)                    v4 = iter(v3)
+                                         v1.setup(v3)   # initialize
         loop start                       loop start
         ...                              ...
         exactly one append per loop      v1.append(..)
         and nothing else done with v2
         ...                              ...
-        loop end                         v2 = hint(v1, {'fence'})
+        loop end                         v2 = v1.fence() or .fence_exact()
     """
     # NB. this assumes RPythonicity: we can only iterate over something
     # that has a len(), and this len() cannot change as long as we are
@@ -672,7 +674,7 @@
     if not newlist_v or not loops:
         return
 
-    # XXX works with Python >= 2.4 only: find calls to append encoded as
+    # find calls to append encoded as
     # getattr/simple_call pairs, as produced by the LIST_APPEND bytecode.
     for block in graph.iterblocks():
         for i in range(len(block.operations)-1):
@@ -805,6 +807,9 @@
         else:
             return None
 
+    def returns_vlist(self, v_result):
+        return self.variable_families.find_rep(v_result) is self.vlistfamily
+
     def remove_vlist(self, args):
         removed = 0
         for i in range(len(args)-1, -1, -1):
@@ -814,6 +819,56 @@
                 removed += 1
         assert removed == 1
 
+    def make_RListCompr_class(self):
+
+        class RListCompr(object):
+            """Class used temporarily for list comprehension, when the
+            list is being built.  In all cases, the instance should be
+            later killed by malloc removal."""
+
+            def setup(self, iterable):
+                # Only optimize iteration over lists or dicts, not over
+                # an iterator object (because it has no known length).
+                # In all cases, 'self.optimize' is an annotation-time
+                # constant.
+                from rpython.rlib import objectmodel
+                self.optimize = objectmodel._is_list_or_dict(iterable)
+                if self.optimize:
+                    self.fixed_list = objectmodel.newlist_fixed(len(iterable))
+                    self.position = 0
+                else:
+                    self.fallback_list = []
+            setup._always_inline_ = True
+
+            def append(self, item):
+                if self.optimize:
+                    p = self.position
+                    self.position = p + 1
+                    self.fixed_list[p] = item
+                else:
+                    self.fallback_list.append(item)
+            append._always_inline_ = True
+
+            def fence_exact(self):
+                if self.optimize:
+                    assert self.position == len(self.fixed_list)
+                    return self.fixed_list
+                else:
+                    return self.fallback_list[:]
+            fence_exact._always_inline_ = True
+
+            def fence(self):
+                if self.optimize:
+                    if self.position == len(self.fixed_list):
+                        return self.fixed_list
+                    else:
+                        return self.fixed_list[:self.position]
+                else:
+                    return self.fallback_list[:]
+            fence._always_inline_ = True
+
+        return RListCompr
+
     def run(self, vlist, vmeth, appendblock):
         # first check that the 'append' method object doesn't escape
         for op in appendblock.operations:
@@ -912,6 +967,19 @@
         for stopblock1 in stopblocks:
             assert stopblock1 not in loopbody
 
+        # Get a fresh copy of the RListCompr class
+        RListCompr = self.make_RListCompr_class()
+
+        # Replace the original newlist() with simple_call(RListCompr)
+        for op in newlistblock.operations:
+            if self.returns_vlist(op.result):
+                assert op.opname == 'newlist'
+                op.opname = 'simple_call'
+                op.args[:] = [Constant(RListCompr)]
+                break
+        else:
+            raise AssertionError("lost 'newlist' operation")
+
         # at StopIteration, the new list is exactly of the same length as
         # the one we iterate over if it's not possible to skip the appendblock
         # in the body:
@@ -919,8 +987,8 @@
                                                 avoid = appendblock,
                                                 stay_within = loopbody)
 
-        # - add a hint(vlist, iterable, {'maxlength'}) in the iterblock,
-        #   where we can compute the known maximum length
+        # - add a 'v2 = getattr(v, 'setup'); simple_call(v2, iterable)'
+        #   in the iterblock, where we can compute the known maximum length
         link = iterblock.exits[0]
         vlist = self.contains_vlist(link.args)
         assert vlist
@@ -930,34 +998,33 @@
                 break
         else:
             raise AssertionError("lost 'iter' operation")
-        vlist2 = Variable(vlist)
-        chint = Constant({'maxlength': True})
+        v_method = Variable()
+        v_none = Variable()
         iterblock.operations += [
-            SpaceOperation('hint', [vlist, op.args[0], chint], vlist2)]
-        link.args = list(link.args)
-        for i in range(len(link.args)):
-            if link.args[i] is vlist:
-                link.args[i] = vlist2
+            SpaceOperation('getattr', [vlist, Constant('setup')], v_method),
+            SpaceOperation('simple_call', [v_method, op.args[0]], v_none)]
 
-        # - wherever the list exits the loop body, add a 'hint({fence})'
+        # - wherever the list exits the loop body, add a '.fence()'
         for block in loopbody:
             for link in block.exits:
                 if link.target not in loopbody:
                     vlist = self.contains_vlist(link.args)
                     if vlist is None:
                         continue  # list not passed along this link anyway
-                    hints = {'fence': True}
+                    name = 'fence'
                     if (exactlength and block is loopnextblock and
                         link.target in stopblocks):
-                        hints['exactlength'] = True
-                    chints = Constant(hints)
+                        name = 'fence_exact'
+                    c_name = Constant(name)
                     newblock = unsimplify.insert_empty_block(None, link)
                     index = link.args.index(vlist)
                     vlist2 = newblock.inputargs[index]
                     vlist3 = Variable(vlist2)
                     newblock.inputargs[index] = vlist3
-                    newblock.operations.append(
-                        SpaceOperation('hint', [vlist3, chints], vlist2))
+                    v_method = Variable()
+                    newblock.operations.extend([
+                        SpaceOperation('getattr', [vlist3, c_name], v_method),
+                        SpaceOperation('simple_call', [v_method], vlist2)])
         # done!
 
 
diff --git a/rpython/translator/test/test_simplify.py b/rpython/translator/test/test_simplify.py
--- a/rpython/translator/test/test_simplify.py
+++ b/rpython/translator/test/test_simplify.py
@@ -244,13 +244,11 @@
         def f1(l):
             return [x*17 for x in l]
         self.check(f1, {
-            'newlist': 1,
             'iter': 1,
             'next': 1,
             'mul':  1,
-            'getattr': 1,
-            'simple_call': 1,
-            'hint': 2,
+            'getattr': 3,       # setup, append, fence_exact
+            'simple_call': 4,   # RListCompr, setup, append, fence_exact
             })
 
     def test_with_exc(self):
@@ -264,12 +262,10 @@
             finally:
                 free_some_stuff()
         self.check(f1, {
-            'newlist': 1,
             'iter': 1,
             'next': 1,
-            'getattr': 1,
-            'simple_call': 4,
-            'hint': 2,
+            'getattr': 3,
+            'simple_call': 7,
             })
 
     def test_canraise_before_iter(self):
@@ -281,13 +277,12 @@
             except ValueError:
                 return []
         self.check(f1, {
-            'newlist': 2,
+            'newlist': 1,
             'iter': 1,
             'next': 1,
             'mul':  1,
-            'getattr': 1,
-            'simple_call': 2,
-            'hint': 2,
+            'getattr': 3,
+            'simple_call': 5,
             })
 
     def test_iterate_over_list(self):
@@ -302,12 +297,10 @@
             return new_l
 
         self.check(f, {
-            'hint': 2,
-            'newlist': 1,
             'iter': 1,
             'next': 1,
-            'getattr': 1,
-            'simple_call': 3,
+            'getattr': 3,
+            'simple_call': 6,
             })
 
 

From noreply at buildbot.pypy.org  Wed Oct 23 12:31:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 23 Oct 2013 12:31:41 +0200 (CEST)
Subject: [pypy-commit] cffi default: Add PyPy support for big-endian
	bitfields
Message-ID: <20131023103141.4FAAC1C1DAE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1379:ab441e95dfbf
Date: 2013-10-23 12:30 +0200
http://bitbucket.org/cffi/cffi/changeset/ab441e95dfbf/

Log:	Add PyPy support for big-endian bitfields

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2960,8 +2960,6 @@
     _test_bitfield_details(flag=2)
 
 def test_bitfield_as_big_endian():
-    if '__pypy__' in sys.builtin_module_names:
-        py.test.skip("no big endian machine supported on pypy for now")
     _test_bitfield_details(flag=4)
 
 

From noreply at buildbot.pypy.org  Wed Oct 23 12:32:21 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 23 Oct 2013 12:32:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: cffi/ab441e95dfbf: add PyPy support for
 big-endian bitfields.
Message-ID: <20131023103221.1FA9C1C1DAE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67525:3ebf9399d5ad
Date: 2013-10-23 12:31 +0200
http://bitbucket.org/pypy/pypy/changeset/3ebf9399d5ad/

Log:	cffi/ab441e95dfbf: add PyPy support for big-endian bitfields.

diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -117,13 +117,17 @@
 
 SF_MSVC_BITFIELDS = 1
 SF_GCC_ARM_BITFIELDS = 2
+SF_GCC_BIG_ENDIAN = 4
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
-elif rffi_platform.getdefined('__arm__', ''):
-    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
-    DEFAULT_SFLAGS = 0
+    if rffi_platform.getdefined('__arm__', ''):
+        DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
+    else:
+        DEFAULT_SFLAGS = 0
+    if sys.byteorder == 'big':
+        DEFAULT_SFLAGS |= SF_GCC_BIG_ENDIAN
 
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
@@ -325,6 +329,9 @@
                     prev_bitfield_free -= fbitsize
                     field_offset_bytes = boffset / 8 - ftype.size
 
+                if sflags & SF_GCC_BIG_ENDIAN:
+                    bitshift = 8 * ftype.size - fbitsize- bitshift
+
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
                                            bitshift, fbitsize)
                 fields_list.append(fld)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2949,8 +2949,6 @@
     _test_bitfield_details(flag=2)
 
 def test_bitfield_as_big_endian():
-    if '__pypy__' in sys.builtin_module_names:
-        py.test.skip("no big endian machine supported on pypy for now")
     _test_bitfield_details(flag=4)
 
 

From noreply at buildbot.pypy.org  Wed Oct 23 13:33:47 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 23 Oct 2013 13:33:47 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: start introducing the rest of the
 barriers: A2V, A2I, etc. (WIP)
Message-ID: <20131023113347.CFF651C3043@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67526:7407d1abd064
Date: 2013-10-22 17:04 +0200
http://bitbucket.org/pypy/pypy/changeset/7407d1abd064/

Log:	start introducing the rest of the barriers: A2V, A2I, etc. (WIP)

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -88,15 +88,21 @@
     field_size = 0
     flag = '\x00'
     stm_dont_track_raw_accesses = False
+    immutable = False
 
     def __init__(self, name, offset, field_size, flag,
-                 stm_dont_track_raw_accesses=False):
+                 stm_dont_track_raw_accesses=False,
+                 immutable=False):
         self.name = name
         self.offset = offset
         self.field_size = field_size
         self.flag = flag
         self.stm_dont_track_raw_accesses = stm_dont_track_raw_accesses
+        self.immutable = immutable
 
+    def is_immutable(self):
+        return self.immutable
+    
     def is_pointer_field(self):
         return self.flag == FLAG_POINTER
 
@@ -125,8 +131,10 @@
         name = '%s.%s' % (STRUCT._name, fieldname)
         stm_dont_track_raw_accesses = STRUCT._hints.get(
             'stm_dont_track_raw_accesses', False)
+        immutable = STRUCT._immutable_field(fieldname)
         fielddescr = FieldDescr(name, offset, size, flag,
-                                stm_dont_track_raw_accesses)
+                                stm_dont_track_raw_accesses,
+                                immutable)
         cachedict = cache.setdefault(STRUCT, {})
         cachedict[fieldname] = fielddescr
         return fielddescr
@@ -169,13 +177,19 @@
     lendescr = None
     flag = '\x00'
     vinfo = None
+    immutable = False
 
-    def __init__(self, basesize, itemsize, lendescr, flag):
+    def __init__(self, basesize, itemsize, lendescr, flag,
+                 immutable=False):
         self.basesize = basesize
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
+        self.immutable = immutable
 
+    def is_immutable(self):
+        return self.immutable
+    
     def is_array_of_pointers(self):
         return self.flag == FLAG_POINTER
 
@@ -208,7 +222,9 @@
         else:
             lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT)
         flag = get_type_flag(ARRAY_INSIDE.OF)
-        arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag)
+        immutable = ARRAY_INSIDE._immutable_field()
+        arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag,
+                                immutable)
         if ARRAY_OR_STRUCT._gckind == 'gc':
             gccache.init_array_descr(ARRAY_OR_STRUCT, arraydescr)
         cache[ARRAY_OR_STRUCT] = arraydescr
@@ -221,12 +237,17 @@
 class InteriorFieldDescr(AbstractDescr):
     arraydescr = ArrayDescr(0, 0, None, '\x00')  # workaround for the annotator
     fielddescr = FieldDescr('', 0, 0, '\x00')
+    immutable = False
 
-    def __init__(self, arraydescr, fielddescr):
+    def __init__(self, arraydescr, fielddescr, immutable=False):
         assert arraydescr.flag == FLAG_STRUCT
         self.arraydescr = arraydescr
         self.fielddescr = fielddescr
+        self.immutable = immutable
 
+    def is_immutable(self):
+        return self.immutable
+    
     def sort_key(self):
         return self.fielddescr.sort_key()
 
@@ -252,7 +273,8 @@
         else:
             REALARRAY = getattr(ARRAY, arrayfieldname)
         fielddescr = get_field_descr(gc_ll_descr, REALARRAY.OF, name)
-        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        immutable = arraydescr.is_immutable() or fielddescr.is_immutable()
+        descr = InteriorFieldDescr(arraydescr, fielddescr, immutable)
         cache[(ARRAY, name, arrayfieldname)] = descr
         return descr
 
diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -596,8 +596,10 @@
     def _setup_write_barrier(self):
         if self.stm:
             self.A2Rdescr = STMReadBarrierDescr(self, 'A2R')
+            self.A2Idescr = STMReadBarrierDescr(self, 'A2R') # XXX
             self.Q2Rdescr = STMReadBarrierDescr(self, 'Q2R')
             self.A2Wdescr = STMWriteBarrierDescr(self, 'A2W')
+            self.A2Vdescr = STMWriteBarrierDescr(self, 'A2W') # XXX
             self.V2Wdescr = STMWriteBarrierDescr(self, 'V2W')
             self.write_barrier_descr = "wbdescr: do not use"
         else:
diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -1,10 +1,10 @@
 from rpython.jit.backend.llsupport.rewrite import GcRewriterAssembler
-from rpython.jit.backend.llsupport.descr import CallDescr
+from rpython.jit.backend.llsupport.descr import (
+    CallDescr, FieldDescr, InteriorFieldDescr, ArrayDescr)
 from rpython.jit.metainterp.resoperation import ResOperation, rop
 from rpython.jit.metainterp.history import BoxPtr, ConstPtr, ConstInt
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.jit.metainterp import history
 
 #
 # STM Support
@@ -42,37 +42,38 @@
         #
         insert_transaction_break = False
         for op in operations:
+            opnum = op.getopnum()
             if not we_are_translated():
                 # only possible in tests:
-                if op.getopnum() in (rop.COND_CALL_STM_B, 
+                if opnum in (rop.COND_CALL_STM_B, 
                                      -124): # FORCE_SPILL
                     self.newops.append(op)
                     continue
-            if op.getopnum() == rop.DEBUG_MERGE_POINT:
+            if opnum == rop.DEBUG_MERGE_POINT:
                 continue
-            if op.getopnum() == rop.INCREMENT_DEBUG_COUNTER:
+            if opnum == rop.INCREMENT_DEBUG_COUNTER:
                 self.newops.append(op)
                 continue
             # ----------  ptr_eq  ----------
-            if op.getopnum() in (rop.PTR_EQ, rop.INSTANCE_PTR_EQ,
-                                 rop.PTR_NE, rop.INSTANCE_PTR_NE):
+            if opnum in (rop.PTR_EQ, rop.INSTANCE_PTR_EQ,
+                         rop.PTR_NE, rop.INSTANCE_PTR_NE):
                 self.handle_ptr_eq(op)
                 continue
             # ----------  guard_class  ----------
-            if op.getopnum() == rop.GUARD_CLASS:
+            if opnum == rop.GUARD_CLASS:
                 assert self.cpu.vtable_offset is None
                 # requires gcremovetypeptr translation option
                 # uses h_tid which doesn't need a read-barrier
                 self.newops.append(op)
                 continue
             # ----------  pure operations needing read-barrier  ----------
-            if op.getopnum() in (rop.GETFIELD_GC_PURE,
-                                 rop.GETARRAYITEM_GC_PURE,
-                                 rop.ARRAYLEN_GC,):
+            if opnum in (rop.GETFIELD_GC_PURE,
+                        rop.GETARRAYITEM_GC_PURE,
+                        rop.ARRAYLEN_GC,):
                 # e.g. getting inst_intval of a W_IntObject that is
                 # currently only a stub needs to first resolve to a 
                 # real object
-                self.handle_category_operations(op, 'R')
+                self.handle_category_operations(op, 'I')
                 continue
             # ----------  pure operations, guards  ----------
             if op.is_always_pure() or op.is_guard() or op.is_ovf():
@@ -81,7 +82,7 @@
                 # insert a transaction break after call_release_gil
                 # in order to commit the inevitable transaction following
                 # it immediately
-                if (op.getopnum() == rop.GUARD_NOT_FORCED
+                if (opnum == rop.GUARD_NOT_FORCED
                     and insert_transaction_break):
                     # insert transaction_break after GUARD after calls
                     self.newops.append(
@@ -95,18 +96,15 @@
 
                 continue
             # ----------  getfields  ----------
-            if op.getopnum() in (rop.GETFIELD_GC,
-                                 rop.GETARRAYITEM_GC,
-                                 rop.GETINTERIORFIELD_GC):
-                self.handle_category_operations(op, 'R')
+            if opnum in (rop.GETFIELD_GC, rop.GETARRAYITEM_GC,
+                        rop.GETINTERIORFIELD_GC):
+                self.handle_getfields(op)
                 continue
             # ----------  setfields  ----------
-            if op.getopnum() in (rop.SETFIELD_GC,
-                                 rop.SETARRAYITEM_GC,
-                                 rop.SETINTERIORFIELD_GC,
-                                 rop.STRSETITEM,
-                                 rop.UNICODESETITEM):
-                self.handle_category_operations(op, 'W')
+            if opnum in (rop.SETFIELD_GC, rop.SETINTERIORFIELD_GC,
+                         rop.SETARRAYITEM_GC, rop.STRSETITEM,
+                         rop.UNICODESETITEM):
+                self.handle_setfields(op)
                 continue
             # ----------  mallocs  ----------
             if op.is_malloc():
@@ -117,19 +115,18 @@
                 self.emitting_an_operation_that_can_collect()
                 self.next_op_may_be_in_new_transaction()
 
-                if (op.getopnum() == rop.CALL_MAY_FORCE or
-                    op.getopnum() == rop.CALL_ASSEMBLER or
-                    op.getopnum() == rop.CALL_RELEASE_GIL):
+                if opnum in (rop.CALL_MAY_FORCE, rop.CALL_ASSEMBLER,
+                             rop.CALL_RELEASE_GIL):
                     # insert more transaction breaks after function
                     # calls since they are likely to return as
                     # inevitable transactions
                     insert_transaction_break = True
                     
-                if op.getopnum() == rop.CALL_RELEASE_GIL:
+                if opnum == rop.CALL_RELEASE_GIL:
                     # self.fallback_inevitable(op)
                     # is done by assembler._release_gil_shadowstack()
                     self.newops.append(op)
-                elif op.getopnum() == rop.CALL_ASSEMBLER:
+                elif opnum == rop.CALL_ASSEMBLER:
                     self.handle_call_assembler(op)
                 else:
                     # only insert become_inevitable if calling a
@@ -143,16 +140,15 @@
                         self.newops.append(op)
                 continue
             # ----------  copystrcontent  ----------
-            if op.getopnum() in (rop.COPYSTRCONTENT,
-                                 rop.COPYUNICODECONTENT):
+            if opnum in (rop.COPYSTRCONTENT, rop.COPYUNICODECONTENT):
                 self.handle_copystrcontent(op)
                 continue
             # ----------  raw getfields and setfields  ----------
-            if op.getopnum() in (rop.GETFIELD_RAW, rop.SETFIELD_RAW):
+            if opnum in (rop.GETFIELD_RAW, rop.SETFIELD_RAW):
                 if self.maybe_handle_raw_accesses(op):
                     continue
             # ----------  labels  ----------
-            if op.getopnum() == rop.LABEL:
+            if opnum == rop.LABEL:
                 self.emitting_an_operation_that_can_collect()
                 self.next_op_may_be_in_new_transaction()
                 
@@ -161,7 +157,7 @@
                 self.newops.append(op)
                 continue
             # ----------  jumps  ----------
-            if op.getopnum() == rop.JUMP:
+            if opnum == rop.JUMP:
                 self.newops.append(
                     ResOperation(rop.STM_TRANSACTION_BREAK,
                                  [ConstInt(1)], None))
@@ -169,15 +165,11 @@
                 self.newops.append(op)
                 continue
             # ----------  finish, other ignored ops  ----------
-            if op.getopnum() in (rop.FINISH,
-                                 rop.FORCE_TOKEN,
-                                 rop.READ_TIMESTAMP,
-                                 rop.MARK_OPAQUE_PTR,
-                                 rop.JIT_DEBUG,
-                                 rop.KEEPALIVE,
-                                 rop.QUASIIMMUT_FIELD,
-                                 rop.RECORD_KNOWN_CLASS,
-                                 ):
+            if opnum in (rop.FINISH, rop.FORCE_TOKEN,
+                        rop.READ_TIMESTAMP, rop.MARK_OPAQUE_PTR,
+                        rop.JIT_DEBUG, rop.KEEPALIVE,
+                        rop.QUASIIMMUT_FIELD, rop.RECORD_KNOWN_CLASS,
+                        ):
                 self.newops.append(op)
                 continue
             # ----------  fall-back  ----------
@@ -220,13 +212,13 @@
                 return gc.V2Wdescr
             return gc.A2Wdescr
         elif to_cat == 'V':
-            return gc.A2Wdescr
+            return gc.A2Vdescr
         elif to_cat == 'R':
             if from_cat >= 'Q':
                 return gc.Q2Rdescr
             return gc.A2Rdescr
         elif to_cat == 'I':
-            return gc.A2Rdescr
+            return gc.A2Idescr
 
     def gen_initialize_tid(self, v_newgcobj, tid):
         GcRewriterAssembler.gen_initialize_tid(self, v_newgcobj, tid)
@@ -269,6 +261,48 @@
         assert isinstance(v, BoxPtr)
         return v
 
+    def handle_getfields(self, op):
+        opnum = op.getopnum()
+        descr = op.getdescr()
+        target_category = 'R'
+        if opnum == rop.GETFIELD_GC:
+            assert isinstance(descr, FieldDescr)
+            if descr.is_immutable():
+                target_category = 'I'
+        elif opnum == rop.GETINTERIORFIELD_GC:
+            assert isinstance(descr, InteriorFieldDescr)
+            if descr.is_immutable():
+                target_category = 'I'
+        elif opnum == rop.GETARRAYITEM_GC:
+            assert isinstance(descr, ArrayDescr)
+            if descr.is_immutable():
+                target_category = 'I'
+                
+        self.handle_category_operations(op, target_category)
+
+    
+    def handle_setfields(self, op):
+        opnum = op.getopnum()
+        descr = op.getdescr()
+        target_category = 'W'
+        if opnum == rop.SETFIELD_GC:
+            assert isinstance(descr, FieldDescr)
+            if not descr.is_pointer_field():
+                target_category = 'V'
+        elif opnum == rop.SETINTERIORFIELD_GC:
+            assert isinstance(descr, InteriorFieldDescr)
+            if not descr.is_pointer_field():
+                target_category = 'V'
+        elif opnum == rop.SETARRAYITEM_GC:
+            assert isinstance(descr, ArrayDescr)
+            if not descr.is_array_of_pointers():
+                target_category = 'V'
+        elif opnum in (rop.STRSETITEM, rop.UNICODESETITEM):
+            target_category = 'V'
+            
+        self.handle_category_operations(op, target_category)
+
+    
     def handle_category_operations(self, op, target_category):
         lst = op.getarglist()
         lst[0] = self.gen_barrier(lst[0], target_category)
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -37,6 +37,14 @@
         tzdescr = get_field_descr(self.gc_ll_descr, T, 'z')
         tydescr = get_field_descr(self.gc_ll_descr, T, 'y')
         #
+        Q = lltype.GcStruct('Q')
+        INTERIOR = lltype.GcArray(('z', lltype.Ptr(Q)))
+        intdescr = get_array_descr(self.gc_ll_descr, INTERIOR)
+        intdescr.tid = 1291
+        intlendescr = intdescr.lendescr
+        intzdescr = get_interiorfield_descr(self.gc_ll_descr,
+                                            INTERIOR, 'z')
+        #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
         adescr.tid = 4321
@@ -87,7 +95,6 @@
         signedframedescr = self.cpu.signedframedescr
         floatframedescr = self.cpu.floatframedescr
         casmdescr.compiled_loop_token = clt
-        tzdescr = None # noone cares
         #
         namespace.update(locals())
         #
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -188,11 +188,12 @@
         """)
 
     def test_invalidate_read_status_after_write_array_interior(self):
-        ops = ['getarrayitem_gc', 'getinteriorfield_gc']
+        ops = [('getarrayitem_gc', 'adescr'),
+                ('getinteriorfield_gc', 'intzdescr')]
         original = """
             [p0, i1, i2]
-            p1 = %s(p0, i1, descr=adescr)
-            p2 = %s(p0, i2, descr=adescr)
+            p1 = %s(p0, i1, descr=%s)
+            p2 = %s(p0, i2, descr=%s)
             p3 = getfield_gc(p1, descr=tzdescr)
             setfield_gc(p2, p0, descr=tzdescr)
             p4 = getfield_gc(p1, descr=tzdescr)
@@ -201,8 +202,8 @@
         rewritten = """
             [p0, i1, i2]
             cond_call_stm_b(p0, descr=A2Rdescr)
-            p1 = %s(p0, i1, descr=adescr)
-            p2 = %s(p0, i2, descr=adescr)
+            p1 = %s(p0, i1, descr=%s)
+            p2 = %s(p0, i2, descr=%s)
             cond_call_stm_b(p1, descr=A2Rdescr)
             p3 = getfield_gc(p1, descr=tzdescr)
             cond_call_stm_b(p2, descr=A2Wdescr)
@@ -212,9 +213,9 @@
             stm_transaction_break(1)
             jump()
         """
-        for op in ops:
-            self.check_rewrite(original % (op, op), 
-                               rewritten % (op, op))
+        for op, descr in ops:
+            self.check_rewrite(original % (op, descr, op, descr),
+                               rewritten % (op, descr, op, descr))
 
     def test_rewrite_write_barrier_after_malloc(self):
         self.check_rewrite("""
@@ -307,14 +308,14 @@
             [p1, p2, i3]
             setfield_gc(p1, p2, descr=tzdescr)
             label(p1, i3)
-            setfield_gc(p1, i3, descr=tydescr)
+            setfield_gc(p1, i3, descr=tydescr) # noptr
             jump(p1)
         """, """
             [p1, p2, i3]
             cond_call_stm_b(p1, descr=A2Wdescr)
             setfield_gc(p1, p2, descr=tzdescr)
             label(p1, i3)
-            cond_call_stm_b(p1, descr=A2Wdescr)
+            cond_call_stm_b(p1, descr=A2Vdescr) # noptr
             setfield_gc(p1, i3, descr=tydescr)
             stm_transaction_break(1)
             jump(p1)
@@ -397,12 +398,12 @@
     def test_rewrite_getinteriorfield_gc(self):
         self.check_rewrite("""
             [p1, i2]
-            i3 = getinteriorfield_gc(p1, i2, descr=adescr)
+            i3 = getinteriorfield_gc(p1, i2, descr=intzdescr)
             jump(i3)
         """, """
             [p1, i2]
             cond_call_stm_b(p1, descr=A2Rdescr)
-            i3 = getinteriorfield_gc(p1, i2, descr=adescr)
+            i3 = getinteriorfield_gc(p1, i2, descr=intzdescr)
             stm_transaction_break(1)
             jump(i3)
         """)
@@ -443,16 +444,16 @@
         # calls inbetween
         self.check_rewrite("""
             [p1]
-            i1 = getfield_gc(p1, descr=tydescr)
+            i1 = getfield_gc(p1, descr=tydescr) # noptr
             i2 = int_add(i1, 1)
-            setfield_gc(p1, i2, descr=tydescr)
+            setfield_gc(p1, i2, descr=tydescr) # noptr
             jump(p1)
         """, """
             [p1]
             cond_call_stm_b(p1, descr=A2Rdescr)
             i1 = getfield_gc(p1, descr=tydescr)
             i2 = int_add(i1, 1)
-            cond_call_stm_b(p1, descr=A2Wdescr)
+            cond_call_stm_b(p1, descr=A2Vdescr)
             setfield_gc(p1, i2, descr=tydescr)
             stm_transaction_break(1)
             jump(p1)
@@ -461,12 +462,12 @@
     def test_setfield_followed_by_getfield(self):
         self.check_rewrite("""
             [p1]
-            setfield_gc(p1, 123, descr=tydescr)
+            setfield_gc(p1, 123, descr=tydescr) # noptr
             p2 = getfield_gc(p1, descr=tzdescr)
             jump(p2)
         """, """
             [p1]
-            cond_call_stm_b(p1, descr=A2Wdescr)
+            cond_call_stm_b(p1, descr=A2Vdescr)
             setfield_gc(p1, 123, descr=tydescr)
             p2 = getfield_gc(p1, descr=tzdescr)
             stm_transaction_break(1)
@@ -502,14 +503,14 @@
             [p1]
             p2 = getfield_gc(p1, descr=tzdescr)
             call(p2, descr=calldescr1)
-            setfield_gc(p1, 5, descr=tydescr)
+            setfield_gc(p1, 5, descr=tydescr) # noptr
             jump(p2)
         """, """
             [p1]
             cond_call_stm_b(p1, descr=A2Rdescr)
             p2 = getfield_gc(p1, descr=tzdescr)
             call(p2, descr=calldescr1)
-            cond_call_stm_b(p1, descr=A2Wdescr)
+            cond_call_stm_b(p1, descr=A2Vdescr)
             setfield_gc(p1, 5, descr=tydescr)
             stm_transaction_break(1)
             jump(p2)
@@ -584,14 +585,14 @@
     def test_rewrite_unrelated_setarrayitem_gcs(self):
         self.check_rewrite("""
             [p1, i1, p2, p3, i3, p4]
-            setarrayitem_gc(p1, i1, p2, descr=adescr)
-            setarrayitem_gc(p3, i3, p4, descr=adescr)
+            setarrayitem_gc(p1, i1, p2, descr=adescr) #noptr
+            setarrayitem_gc(p3, i3, p4, descr=adescr) #noptr
             jump()
         """, """
             [p1, i1, p2, p3, i3, p4]
-            cond_call_stm_b(p1, descr=A2Wdescr)
+            cond_call_stm_b(p1, descr=A2Vdescr)
             setarrayitem_gc(p1, i1, p2, descr=adescr)
-            cond_call_stm_b(p3, descr=A2Wdescr)
+            cond_call_stm_b(p3, descr=A2Vdescr)
             setarrayitem_gc(p3, i3, p4, descr=adescr)
             stm_transaction_break(1)
             jump()
@@ -600,13 +601,13 @@
     def test_rewrite_several_setarrayitem_gcs(self):
         self.check_rewrite("""
             [p1, p2, i2, p3, i3]
-            setarrayitem_gc(p1, i2, p2, descr=adescr)
+            setarrayitem_gc(p1, i2, p2, descr=adescr) #noptr
             i4 = read_timestamp()
-            setarrayitem_gc(p1, i3, p3, descr=adescr)
+            setarrayitem_gc(p1, i3, p3, descr=adescr) #noptr
             jump()
         """, """
             [p1, p2, i2, p3, i3]
-            cond_call_stm_b(p1, descr=A2Wdescr)
+            cond_call_stm_b(p1, descr=A2Vdescr)
             setarrayitem_gc(p1, i2, p2, descr=adescr)
             i4 = read_timestamp()
             setarrayitem_gc(p1, i3, p3, descr=adescr)
@@ -617,16 +618,16 @@
     def test_rewrite_several_setinteriorfield_gc(self):
         self.check_rewrite("""
             [p1, p2, i2, p3, i3]
-            setinteriorfield_gc(p1, i2, p2, descr=adescr)
+            setinteriorfield_gc(p1, i2, p2, descr=intzdescr)
             i4 = read_timestamp()
-            setinteriorfield_gc(p1, i3, p3, descr=adescr)
+            setinteriorfield_gc(p1, i3, p3, descr=intzdescr)
             jump()
         """, """
             [p1, p2, i2, p3, i3]
             cond_call_stm_b(p1, descr=A2Wdescr)
-            setinteriorfield_gc(p1, i2, p2, descr=adescr)
+            setinteriorfield_gc(p1, i2, p2, descr=intzdescr)
             i4 = read_timestamp()
-            setinteriorfield_gc(p1, i3, p3, descr=adescr)
+            setinteriorfield_gc(p1, i3, p3, descr=intzdescr)
             stm_transaction_break(1)
             jump()
         """)
@@ -639,7 +640,7 @@
             jump()
         """, """
             [p1, i2, i3]
-            cond_call_stm_b(p1, descr=A2Wdescr)
+            cond_call_stm_b(p1, descr=A2Vdescr)
             strsetitem(p1, i2, i3)
             unicodesetitem(p1, i2, i3)
             stm_transaction_break(1)
@@ -653,19 +654,19 @@
         calldescr2 = get_call_descr(self.gc_ll_descr, [T], rffi.TIME_T)
         self.check_rewrite("""
             [i1, i2, i3, p7]
-            setfield_gc(p7, 10, descr=tydescr)
+            setfield_gc(p7, 10, descr=tydescr) #noptr
             call_release_gil(123, descr=calldescr2)
             guard_not_forced() []
-            setfield_gc(p7, 20, descr=tydescr)
+            setfield_gc(p7, 20, descr=tydescr) #noptr
             jump(i2, p7)
         """, """
             [i1, i2, i3, p7]
-            cond_call_stm_b(p7, descr=A2Wdescr)
+            cond_call_stm_b(p7, descr=A2Vdescr)
             setfield_gc(p7, 10, descr=tydescr)
             call_release_gil(123, descr=calldescr2)
             guard_not_forced() []
             stm_transaction_break(0)
-            cond_call_stm_b(p7, descr=A2Wdescr)
+            cond_call_stm_b(p7, descr=A2Vdescr)
             setfield_gc(p7, 20, descr=tydescr)
             stm_transaction_break(1)
             jump(i2, p7)
@@ -677,7 +678,7 @@
         oplist = [
             "setfield_raw(i1, i2, descr=tydescr)",
             "setarrayitem_raw(i1, i2, i3, descr=tydescr)",
-            "setinteriorfield_raw(i1, i2, i3, descr=adescr)",
+            "setinteriorfield_raw(i1, i2, i3, descr=intzdescr)",
             "escape(i1)",    # a generic unknown operation
             ]
         for op in oplist:
@@ -689,11 +690,11 @@
                 jump(i2, p7)
             """ % op, """
                 [i1, i2, i3, p7]
-                cond_call_stm_b(p7, descr=A2Wdescr)
+                cond_call_stm_b(p7, descr=A2Vdescr)
                 setfield_gc(p7, 10, descr=tydescr)
                 $INEV
                 %s
-                cond_call_stm_b(p7, descr=A2Wdescr)
+                cond_call_stm_b(p7, descr=A2Vdescr)
                 setfield_gc(p7, 20, descr=tydescr)
                 stm_transaction_break(1)
                 jump(i2, p7)
@@ -758,18 +759,20 @@
                 jump(p1)
             """ % (op, guard), """
                 [p1]
-                cond_call_stm_b(p1, descr=A2Wdescr)
+                cond_call_stm_b(p1, descr=A2Vdescr)
                 setfield_gc(p1, 10, descr=tydescr)
                 %s
                 %s
                 %s
-                cond_call_stm_b(p1, descr=A2Wdescr)
+                cond_call_stm_b(p1, descr=A2Vdescr)
                 setfield_gc(p1, 20, descr=tydescr)
                 stm_transaction_break(1)
                 jump(p1)
             """ % (op, guard, tr_break), calldescr2=calldescr2)
 
     def test_call_assembler(self):
+        py.test.skip("XXX: works, but somehow the test doesn't")
+
         self.check_rewrite("""
         [i0, f0]
         i2 = call_assembler(i0, f0, descr=casmdescr)

From noreply at buildbot.pypy.org  Wed Oct 23 13:33:49 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 23 Oct 2013 13:33:49 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: add tests for immutable getfields and
	noptr setfields
Message-ID: <20131023113349.391BF1C3043@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67527:afce459b2117
Date: 2013-10-22 17:47 +0200
http://bitbucket.org/pypy/pypy/changeset/afce459b2117/

Log:	add tests for immutable getfields and noptr setfields

diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -1227,6 +1227,76 @@
             jump()
         """, calldescr2=calldescr2)
 
-        
+    def test_immutable_getfields(self):
+        for imm_hint in [{}, {'immutable':True}]:
+            S = lltype.GcStruct('S')
+            U = lltype.GcStruct('U',
+                ('x', lltype.Signed),
+                ('y', lltype.Ptr(S)),
+                hints=imm_hint)
+            udescr = get_size_descr(self.gc_ll_descr, U)
+            udescr.tid = 2123
+            uxdescr = get_field_descr(self.gc_ll_descr, U, 'x')
+            #uydescr = get_field_descr(self.gc_ll_descr, U, 'y')
 
+            V = lltype.GcArray(('z', lltype.Ptr(S)), hints=imm_hint)
+            vdescr = get_array_descr(self.gc_ll_descr, V)
+            vdescr.tid = 1233
+            vzdescr = get_interiorfield_descr(self.gc_ll_descr, V, 'z')
 
+            barr = "A2Idescr" if imm_hint else "A2Rdescr"
+            self.check_rewrite("""
+                [p1, p3, i1, p4]
+                p2 = getfield_gc(p1, descr=uxdescr)
+                i3 = getinteriorfield_gc(p3, i1, descr=vzdescr)
+                i4 = getarrayitem_gc(p4, i3, descr=vdescr)
+                jump(p2)
+            """, """
+                [p1, p3, i1, p4]
+                cond_call_stm_b(p1, descr=%s)
+                p2 = getfield_gc(p1, descr=uxdescr)
+                cond_call_stm_b(p3, descr=%s)
+                i3 = getinteriorfield_gc(p3, i1, descr=vzdescr)
+                cond_call_stm_b(p4, descr=%s)
+                i4 = getarrayitem_gc(p4, i3, descr=vdescr)
+                stm_transaction_break(1)
+                jump(p2)
+            """ % (barr, barr, barr), uxdescr=uxdescr,
+            vzdescr=vzdescr, vdescr=vdescr)
+
+    def test_noptr_setfields(self):
+        S = lltype.GcStruct('S')
+        U = lltype.GcStruct('U',
+                ('x', lltype.Signed),
+                ('y', lltype.Ptr(S)))
+        udescr = get_size_descr(self.gc_ll_descr, U)
+        udescr.tid = 2123
+        uxdescr = get_field_descr(self.gc_ll_descr, U, 'x')
+        #uydescr = get_field_descr(self.gc_ll_descr, U, 'y')
+
+        V = lltype.GcArray(('z', lltype.Signed))
+        vdescr = get_array_descr(self.gc_ll_descr, V)
+        vdescr.tid = 1233
+        vzdescr = get_interiorfield_descr(self.gc_ll_descr, V, 'z')
+
+        self.check_rewrite("""
+                [p1, p3, i1, p4]
+                setfield_gc(p1, 1, descr=uxdescr)
+                setinteriorfield_gc(p3, i1, 1, descr=vzdescr)
+                setarrayitem_gc(p4, i1, 1, descr=vdescr)
+                jump(p3)
+            """, """
+                [p1, p3, i1, p4]
+                cond_call_stm_b(p1, descr=A2Vdescr)
+                setfield_gc(p1, 1, descr=uxdescr)
+                cond_call_stm_b(p3, descr=A2Vdescr)
+                setinteriorfield_gc(p3, i1, 1, descr=vzdescr)
+                cond_call_stm_b(p4, descr=A2Vdescr)
+                setarrayitem_gc(p4, i1, 1, descr=vdescr)
+                stm_transaction_break(1)
+                jump(p3)
+            """, uxdescr=uxdescr, vzdescr=vzdescr, vdescr=vdescr)
+
+    
+
+

From noreply at buildbot.pypy.org  Wed Oct 23 13:33:50 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 23 Oct 2013 13:33:50 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: implement the additional barriers with
 fastpaths in assembler.py and gc.py
Message-ID: <20131023113350.7B4411C3043@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67528:2a83d4ee265b
Date: 2013-10-23 13:32 +0200
http://bitbucket.org/pypy/pypy/changeset/2a83d4ee265b/

Log:	implement the additional barriers with fastpaths in assembler.py and
	gc.py

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -82,6 +82,7 @@
         self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn)
         if gc_ll_descr.stm:
             descrs = [gc_ll_descr.A2Rdescr, gc_ll_descr.Q2Rdescr,
+                      gc_ll_descr.A2Idescr, gc_ll_descr.A2Vdescr,
                       gc_ll_descr.A2Wdescr, gc_ll_descr.V2Wdescr]
         else:
             descrs = [gc_ll_descr.write_barrier_descr]
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -88,7 +88,7 @@
     field_size = 0
     flag = '\x00'
     stm_dont_track_raw_accesses = False
-    immutable = False
+    _immutable = False
 
     def __init__(self, name, offset, field_size, flag,
                  stm_dont_track_raw_accesses=False,
@@ -98,10 +98,10 @@
         self.field_size = field_size
         self.flag = flag
         self.stm_dont_track_raw_accesses = stm_dont_track_raw_accesses
-        self.immutable = immutable
+        self._immutable = immutable
 
     def is_immutable(self):
-        return self.immutable
+        return self._immutable
     
     def is_pointer_field(self):
         return self.flag == FLAG_POINTER
@@ -131,7 +131,7 @@
         name = '%s.%s' % (STRUCT._name, fieldname)
         stm_dont_track_raw_accesses = STRUCT._hints.get(
             'stm_dont_track_raw_accesses', False)
-        immutable = STRUCT._immutable_field(fieldname)
+        immutable = bool(STRUCT._immutable_field(fieldname))
         fielddescr = FieldDescr(name, offset, size, flag,
                                 stm_dont_track_raw_accesses,
                                 immutable)
@@ -177,7 +177,7 @@
     lendescr = None
     flag = '\x00'
     vinfo = None
-    immutable = False
+    _immutable = False
 
     def __init__(self, basesize, itemsize, lendescr, flag,
                  immutable=False):
@@ -185,10 +185,10 @@
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
-        self.immutable = immutable
+        self._immutable = immutable
 
     def is_immutable(self):
-        return self.immutable
+        return self._immutable
     
     def is_array_of_pointers(self):
         return self.flag == FLAG_POINTER
@@ -222,7 +222,7 @@
         else:
             lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT)
         flag = get_type_flag(ARRAY_INSIDE.OF)
-        immutable = ARRAY_INSIDE._immutable_field()
+        immutable = bool(ARRAY_INSIDE._immutable_field())
         arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag,
                                 immutable)
         if ARRAY_OR_STRUCT._gckind == 'gc':
@@ -237,16 +237,16 @@
 class InteriorFieldDescr(AbstractDescr):
     arraydescr = ArrayDescr(0, 0, None, '\x00')  # workaround for the annotator
     fielddescr = FieldDescr('', 0, 0, '\x00')
-    immutable = False
+    _immutable = False
 
     def __init__(self, arraydescr, fielddescr, immutable=False):
         assert arraydescr.flag == FLAG_STRUCT
         self.arraydescr = arraydescr
         self.fielddescr = fielddescr
-        self.immutable = immutable
+        self._immutable = immutable
 
     def is_immutable(self):
-        return self.immutable
+        return self._immutable
     
     def sort_key(self):
         return self.fielddescr.sort_key()
@@ -273,7 +273,7 @@
         else:
             REALARRAY = getattr(ARRAY, arrayfieldname)
         fielddescr = get_field_descr(gc_ll_descr, REALARRAY.OF, name)
-        immutable = arraydescr.is_immutable() or fielddescr.is_immutable()
+        immutable = bool(arraydescr.is_immutable() or fielddescr.is_immutable())
         descr = InteriorFieldDescr(arraydescr, fielddescr, immutable)
         cache[(ARRAY, name, arrayfieldname)] = descr
         return descr
diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -428,13 +428,14 @@
 
 class STMReadBarrierDescr(STMBarrierDescr):
     def __init__(self, gc_ll_descr, stmcat):
-        assert stmcat in ['A2R', 'Q2R']
-        if stmcat == 'A2R':
-            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
-                                     'stm_DirectReadBarrier')
-        else:
-            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
-                                     'stm_RepeatReadBarrier')
+        assert stmcat in ['A2R', 'Q2R', 'A2I']
+        func = {'A2R': 'stm_DirectReadBarrier',
+                'Q2R': 'stm_RepeatReadBarrier',
+                'A2I': 'stm_ImmutReadBarrier',
+                }
+        
+        STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
+                                 func[stmcat])
 
     @specialize.arg(2)
     def _do_barrier(self, gcref_struct, returns_modified_object):
@@ -458,12 +459,16 @@
             rcp = rffi.cast(CP, read_cache[0])
             if rcp[index] == objint:
                 return gcref_struct
-        else: # 'Q2R'
+        elif self.stmcat == 'Q2R':
             # is GCFLAG_PUBLIC_TO_PRIVATE or GCFLAG_MOVED set?
             if not (objhdr.h_tid &
                     (StmGC.GCFLAG_PUBLIC_TO_PRIVATE | StmGC.GCFLAG_MOVED)):
                 # no.
                 return gcref_struct
+        else: # A2I
+            # GCFLAG_STUB set?
+            if not (objhdr.h_tid & StmGC.GCFLAG_STUB):
+                return gcref_struct
         
         funcptr = self.get_barrier_funcptr(returns_modified_object)
         res = funcptr(objadr)
@@ -472,13 +477,14 @@
         
 class STMWriteBarrierDescr(STMBarrierDescr):
     def __init__(self, gc_ll_descr, stmcat):
-        assert stmcat in ['A2W', 'V2W']
-        if stmcat == 'A2W':
-            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
-                                     'stm_WriteBarrier')
-        else:
-            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
-                                     'stm_RepeatWriteBarrier')
+        assert stmcat in ['A2W', 'V2W', 'A2V']
+        func = {'A2W':'stm_WriteBarrier',
+                'V2W':'stm_RepeatWriteBarrier',
+                'A2V':'stm_WriteBarrier',
+                }
+
+        STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
+                                 func[stmcat])
 
 
     @specialize.arg(2)
@@ -487,12 +493,17 @@
         from rpython.memory.gc.stmgc import StmGC
         objadr = llmemory.cast_ptr_to_adr(gcref_struct)
         objhdr = rffi.cast(StmGC.GCHDRP, gcref_struct)
+
+        # for A2W, we check h_revision and WRITE_BARRIER
+        # for V2W, we only check WRITE_BARRIER
+        # for A2V, we only check h_revision
         
         # if it is a repeated WB or h_revision == privat_rev of transaction
         priv_rev = self.llop1.stm_get_adr_of_private_rev_num(rffi.SIGNEDP)
         if self.stmcat == 'V2W' or objhdr.h_revision == priv_rev[0]:
             # also WRITE_BARRIER not set?
-            if not (objhdr.h_tid & StmGC.GCFLAG_WRITE_BARRIER):
+            if (self.stmcat == 'A2V'
+                or not (objhdr.h_tid & StmGC.GCFLAG_WRITE_BARRIER)):
                 return gcref_struct
         
         funcptr = self.get_barrier_funcptr(returns_modified_object)
@@ -596,10 +607,10 @@
     def _setup_write_barrier(self):
         if self.stm:
             self.A2Rdescr = STMReadBarrierDescr(self, 'A2R')
-            self.A2Idescr = STMReadBarrierDescr(self, 'A2R') # XXX
+            self.A2Idescr = STMReadBarrierDescr(self, 'A2I')
             self.Q2Rdescr = STMReadBarrierDescr(self, 'Q2R')
             self.A2Wdescr = STMWriteBarrierDescr(self, 'A2W')
-            self.A2Vdescr = STMWriteBarrierDescr(self, 'A2W') # XXX
+            self.A2Vdescr = STMWriteBarrierDescr(self, 'A2V')
             self.V2Wdescr = STMWriteBarrierDescr(self, 'V2W')
             self.write_barrier_descr = "wbdescr: do not use"
         else:
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -45,6 +45,25 @@
             descr_s = get_size_descr(c0, STRUCT)
             assert descr_s.count_fields_if_immutable() == expected
 
+def test_is_immutable():
+    U = lltype.Struct('U', ('x', lltype.Char),
+                      hints={'immutable':True})
+    V = lltype.Struct('V', ('x', lltype.Char))
+    gc = GcCache(False)
+    assert get_field_descr(gc, U, 'x').is_immutable()
+    assert not get_field_descr(gc, V, 'x').is_immutable()
+
+    A1 = lltype.GcArray(lltype.Char, hints={'immutable':True})
+    A2 = lltype.GcArray(lltype.Char)
+    assert get_array_descr(gc, A1).is_immutable()
+    assert not get_array_descr(gc, A2).is_immutable()
+
+    I1 = lltype.GcArray(('z', lltype.Char), hints={'immutable':True})
+    I2 = lltype.GcArray(('z', lltype.Char))
+    assert get_interiorfield_descr(gc, I1, 'z').is_immutable()
+    assert not get_interiorfield_descr(gc, I2, 'z').is_immutable()
+    
+    
 def test_get_field_descr():
     U = lltype.Struct('U')
     T = lltype.GcStruct('T')
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2489,24 +2489,28 @@
             helper_num += 2
         #
         # FASTPATH:
-        #
+        # do slowpath IF:
         # A2W:
         # (obj->h_revision != stm_private_rev_num)
         #     || (obj->h_tid & GCFLAG_WRITE_BARRIER) != 0)
         # V2W:
         # (obj->h_tid & GCFLAG_WRITE_BARRIER) != 0)
+        # A2V:
+        # (obj->h_revision != stm_private_rev_num)
         # A2R:
         # (obj->h_revision != stm_private_rev_num)
         #     && (FXCACHE_AT(obj) != obj)))
         # Q2R:
         # (obj->h_tid & (GCFLAG_PUBLIC_TO_PRIVATE | GCFLAG_MOVED) != 0)
+        # A2I:
+        # (obj->h_tid & GCFLAG_STUB)
         if IS_X86_32:   # XXX: todo
             todo()
         jz_location = 0
         jz_location2 = 0
         jnz_location = 0
         # compare h_revision with stm_private_rev_num
-        if descr.stmcat in ['A2W', 'A2R']:
+        if descr.stmcat in ['A2W', 'A2R', 'A2V']:
             rn = self._get_stm_private_rev_num_addr()
             if we_are_translated():
                 # during tests, _get_stm_private_rev_num_addr returns
@@ -2521,11 +2525,11 @@
             else:
                 mc.CMP(X86_64_SCRATCH_REG, mem(loc_base, StmGC.H_REVISION))
             #
-            if descr.stmcat == 'A2R':
+            if descr.stmcat in ('A2R', 'A2V'):
                 # jump to end if h_rev==priv_rev
                 mc.J_il8(rx86.Conditions['Z'], 0) # patched below
                 jz_location = mc.get_relative_pos()
-            else: # write_barrier
+            else: # A2W
                 # jump to slowpath if h_rev!=priv_rev
                 mc.J_il8(rx86.Conditions['NZ'], 0) # patched below
                 jnz_location = mc.get_relative_pos()
@@ -2554,21 +2558,27 @@
             jz_location2 = mc.get_relative_pos()
         #
         # check flags:
-        if descr.stmcat in ['A2W', 'V2W', 'Q2R']:
+        if descr.stmcat in ['A2W', 'V2W', 'Q2R', 'A2I']:
             flags = 0
+            off = 0
             if descr.stmcat in ['A2W', 'V2W']:
                 # obj->h_tid & GCFLAG_WRITE_BARRIER) != 0
-                assert IS_X86_64 and (StmGC.GCFLAG_WRITE_BARRIER >> 32) > 0
-                assert (StmGC.GCFLAG_WRITE_BARRIER >> 40) == 0
-                flags = StmGC.GCFLAG_WRITE_BARRIER >> 32
+                flags = StmGC.GCFLAG_WRITE_BARRIER
             elif descr.stmcat == 'Q2R':
                 # obj->h_tid & PUBLIC_TO_PRIVATE|MOVED
                 flags = StmGC.GCFLAG_PUBLIC_TO_PRIVATE | StmGC.GCFLAG_MOVED
-                assert IS_X86_64 and (flags >> 32) > 0
-                assert (flags >> 40) == 0
+            elif descr.stmcat == 'A2I':
+                # obj->h_tid & STUB
+                flags = StmGC.GCFLAG_STUB
+
+            assert IS_X86_64
+            if (flags >> 32) > 0 and (flags >> 40) == 0:
                 flags = flags >> 32
-
-            off = 4
+                off = 4
+            elif (flags >> 40) > 0 and (flags >> 48) == 0:
+                flags = flags >> 40
+                off = 5
+            #
             if loc_base == ebp:
                 mc.TEST8_bi(StmGC.H_TID + off, flags)
             else:
diff --git a/rpython/jit/backend/x86/test/test_stm_integration.py b/rpython/jit/backend/x86/test/test_stm_integration.py
--- a/rpython/jit/backend/x86/test/test_stm_integration.py
+++ b/rpython/jit/backend/x86/test/test_stm_integration.py
@@ -153,8 +153,10 @@
             return obj
 
         self.A2Rdescr = FakeSTMBarrier(self, 'A2R', read_barrier)
+        self.A2Idescr = FakeSTMBarrier(self, 'A2I', read_barrier)
         self.Q2Rdescr = FakeSTMBarrier(self, 'Q2R', read_barrier)
         self.A2Wdescr = FakeSTMBarrier(self, 'A2W', write_barrier)
+        self.A2Vdescr = FakeSTMBarrier(self, 'A2V', write_barrier)
         self.V2Wdescr = FakeSTMBarrier(self, 'V2W', write_barrier)
         
         self.do_write_barrier = None
@@ -256,8 +258,10 @@
         
 
         self.a2wd = cpu.gc_ll_descr.A2Wdescr
+        self.a2vd = cpu.gc_ll_descr.A2Vdescr
         self.v2wd = cpu.gc_ll_descr.V2Wdescr
         self.a2rd = cpu.gc_ll_descr.A2Rdescr
+        self.a2id = cpu.gc_ll_descr.A2Idescr
         self.q2rd = cpu.gc_ll_descr.Q2Rdescr
 
         TP = rffi.CArray(lltype.Signed)
@@ -391,6 +395,39 @@
             else:
                 self.assert_in(called, [sgcref])
 
+    def test_gc_immutable_read_barrier_fastpath(self):
+        from rpython.jit.backend.llsupport.gc import STMReadBarrierDescr
+        descr = STMReadBarrierDescr(self.cpu.gc_ll_descr, 'A2I')
+
+        called = []
+        def read(obj):
+            called.append(obj)
+            return obj
+
+        functype = lltype.Ptr(lltype.FuncType(
+            [llmemory.Address], llmemory.Address))
+        funcptr = llhelper(functype, read)
+        descr.b_failing_case_ptr = funcptr
+        descr.llop1 = fakellop()
+
+        # -------- TEST --------
+        for flags in [StmGC.GCFLAG_STUB, 0]:
+            called[:] = []
+            
+            s = self.allocate_prebuilt_s()
+            sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+            s.h_tid |= flags
+
+            descr._do_barrier(sgcref, returns_modified_object=True)
+                        
+            # check if rev-fastpath worked
+            if not flags:
+                # fastpath
+                self.assert_not_in(called, [sgcref])
+            else:
+                self.assert_in(called, [sgcref])
+
+
 
     def test_gc_write_barrier_fastpath(self):
         from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr
@@ -462,9 +499,50 @@
         descr._do_barrier(sgcref, 
             returns_modified_object=True)
         self.assert_in(called, [sgcref])
-        
-        
-        
+
+    def test_gc_noptr_write_barrier_fastpath(self):
+        from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr
+        descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'A2V')
+
+        called = []
+        def write(obj):
+            called.append(obj)
+            return obj
+
+        functype = lltype.Ptr(lltype.FuncType(
+            [llmemory.Address], llmemory.Address))
+        funcptr = llhelper(functype, write)
+        descr.b_failing_case_ptr = funcptr
+        descr.llop1 = fakellop()
+
+        # -------- TEST --------
+        for rev in [fakellop.PRIV_REV+4, fakellop.PRIV_REV]:
+            called[:] = []
+            
+            s = self.allocate_prebuilt_s()
+            sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+            s.h_revision = rev
+
+            descr._do_barrier(sgcref, returns_modified_object=True)
+                        
+            # check if fastpath worked
+            if rev == fakellop.PRIV_REV:
+                # fastpath
+                self.assert_not_in(called, [sgcref])
+            else:
+                self.assert_in(called, [sgcref])
+                
+            # now set WRITE_BARRIER -> no effect
+            called[:] = []
+            s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER
+            descr._do_barrier(sgcref, returns_modified_object=True)
+            if rev == fakellop.PRIV_REV:
+                # fastpath
+                self.assert_not_in(called, [sgcref])
+            else:
+                self.assert_in(called, [sgcref])
+
+                        
         
     def test_read_barrier_fastpath(self):
         cpu = self.cpu
@@ -542,6 +620,40 @@
             else:
                 self.assert_in(called_on, [sgcref])
 
+    def test_immutable_read_barrier_fastpath(self):
+        cpu = self.cpu
+        cpu.gc_ll_descr.init_nursery(100)
+        cpu.setup_once()
+
+        called_on = cpu.gc_ll_descr.rb_called_on
+        for flags in [StmGC.GCFLAG_STUB, 0]:
+            cpu.gc_ll_descr.clear_lists()
+            self.clear_read_cache()
+            
+            s = self.allocate_prebuilt_s()
+            sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+            s.h_tid |= flags
+            
+            p0 = BoxPtr()
+            operations = [
+                ResOperation(rop.COND_CALL_STM_B, [p0], None,
+                             descr=self.a2id),
+                ResOperation(rop.FINISH, [p0], None, 
+                             descr=BasicFinalDescr(0)),
+                ]
+            inputargs = [p0]
+            looptoken = JitCellToken()
+            cpu.compile_loop(None, inputargs, operations, looptoken)
+            self.cpu.execute_token(looptoken, sgcref)
+            
+            # check if rev-fastpath worked
+            if not flags:
+                # fastpath
+                self.assert_not_in(called_on, [sgcref])
+            else:
+                self.assert_in(called_on, [sgcref])
+
+
 
     def test_write_barrier_fastpath(self):
         cpu = self.cpu
@@ -616,6 +728,50 @@
         self.cpu.execute_token(looptoken, sgcref)
         self.assert_in(called_on, [sgcref])
 
+    def test_noptr_write_barrier_fastpath(self):
+        cpu = self.cpu
+        cpu.gc_ll_descr.init_nursery(100)
+        cpu.setup_once()
+        PRIV_REV = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION)
+        self.priv_rev_num[0] = PRIV_REV
+        called_on = cpu.gc_ll_descr.wb_called_on
+        
+        for rev in [PRIV_REV+4, PRIV_REV]:
+            cpu.gc_ll_descr.clear_lists()
+            
+            s = self.allocate_prebuilt_s()
+            sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+            s.h_revision = rev
+            
+            p0 = BoxPtr()
+            operations = [
+                ResOperation(rop.COND_CALL_STM_B, [p0], None,
+                             descr=self.a2vd),
+                ResOperation(rop.FINISH, [p0], None, 
+                             descr=BasicFinalDescr(0)),
+                ]
+            inputargs = [p0]
+            looptoken = JitCellToken()
+            cpu.compile_loop(None, inputargs, operations, looptoken)
+            self.cpu.execute_token(looptoken, sgcref)
+            
+            # check if rev-fastpath worked
+            if rev == PRIV_REV:
+                # fastpath and WRITE_BARRIER not set
+                self.assert_not_in(called_on, [sgcref])
+            else:
+                self.assert_in(called_on, [sgcref])
+
+            # now set WRITE_BARRIER -> no effect
+            cpu.gc_ll_descr.clear_lists()
+            s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER
+            self.cpu.execute_token(looptoken, sgcref)
+            if rev == PRIV_REV:
+                # fastpath and WRITE_BARRIER not set
+                self.assert_not_in(called_on, [sgcref])
+            else:
+                self.assert_in(called_on, [sgcref])
+
             
     def test_ptr_eq_fastpath(self):
         cpu = self.cpu

From noreply at buildbot.pypy.org  Wed Oct 23 17:42:04 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 23 Oct 2013 17:42:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Include teh location of the abort
Message-ID: <20131023154204.65B741C3085@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67529:826d0ebef267
Date: 2013-10-22 16:25 -0700
http://bitbucket.org/pypy/pypy/changeset/826d0ebef267/

Log:	Include teh location of the abort

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1922,7 +1922,7 @@
 
     def aborted_tracing(self, reason):
         self.staticdata.profiler.count(reason)
-        debug_print('~~~ ABORTING TRACING')
+        debug_print('~~~ ABORTING TRACING inside: %s' % self.framestack[-1].jitcode.name)
         jd_sd = self.jitdriver_sd
         if not self.current_merge_points:
             greenkey = None # we're in the bridge

From noreply at buildbot.pypy.org  Wed Oct 23 17:42:05 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 23 Oct 2013 17:42:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: Tiny cleanup
Message-ID: <20131023154205.8F2791C30A6@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67530:0ba98e5aff4f
Date: 2013-10-22 16:27 -0700
http://bitbucket.org/pypy/pypy/changeset/0ba98e5aff4f/

Log:	Tiny cleanup

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -328,7 +328,6 @@
         return misc.longdouble2str(lvalue)
 
     def cast(self, w_ob):
-        space = self.space
         if (isinstance(w_ob, cdataobj.W_CData) and
                 isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)):
             w_cdata = self.convert_to_object(w_ob._cdata)

From noreply at buildbot.pypy.org  Wed Oct 23 17:42:06 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 23 Oct 2013 17:42:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: Put this logging info in a better place
Message-ID: <20131023154206.BEC4D1C30DC@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67531:95b246d47b69
Date: 2013-10-23 08:07 -0700
http://bitbucket.org/pypy/pypy/changeset/95b246d47b69/

Log:	Put this logging info in a better place

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1922,7 +1922,7 @@
 
     def aborted_tracing(self, reason):
         self.staticdata.profiler.count(reason)
-        debug_print('~~~ ABORTING TRACING inside: %s' % self.framestack[-1].jitcode.name)
+        debug_print('~~~ ABORTING TRACING')
         jd_sd = self.jitdriver_sd
         if not self.current_merge_points:
             greenkey = None # we're in the bridge
@@ -2445,6 +2445,7 @@
             if vinfo.tracing_after_residual_call(virtualizable):
                 # the virtualizable escaped during CALL_MAY_FORCE.
                 self.load_fields_from_virtualizable()
+                debug_print('vable escaped during a call in: %s' % self.framestack[-1].jitcode.name)
                 raise SwitchToBlackhole(Counters.ABORT_ESCAPE,
                                         raising_exception=True)
                 # ^^^ we set 'raising_exception' to True because we must still

From noreply at buildbot.pypy.org  Wed Oct 23 17:42:08 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 23 Oct 2013 17:42:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20131023154208.1B2561C30E8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67532:fe8d08ae6823
Date: 2013-10-23 08:41 -0700
http://bitbucket.org/pypy/pypy/changeset/fe8d08ae6823/

Log:	merged upstream

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -111,3 +111,6 @@
 
 .. branch: incremental-gc
 Added the new incminimark GC which performs GC in incremental steps
+
+.. branch: fast_cffi_list_init
+fastpath for cffi.new("long[]")
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -117,13 +117,17 @@
 
 SF_MSVC_BITFIELDS = 1
 SF_GCC_ARM_BITFIELDS = 2
+SF_GCC_BIG_ENDIAN = 4
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
-elif rffi_platform.getdefined('__arm__', ''):
-    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
-    DEFAULT_SFLAGS = 0
+    if rffi_platform.getdefined('__arm__', ''):
+        DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
+    else:
+        DEFAULT_SFLAGS = 0
+    if sys.byteorder == 'big':
+        DEFAULT_SFLAGS |= SF_GCC_BIG_ENDIAN
 
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
@@ -325,6 +329,9 @@
                     prev_bitfield_free -= fbitsize
                     field_offset_bytes = boffset / 8 - ftype.size
 
+                if sflags & SF_GCC_BIG_ENDIAN:
+                    bitshift = 8 * ftype.size - fbitsize- bitshift
+
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
                                            bitshift, fbitsize)
                 fields_list.append(fld)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2949,8 +2949,6 @@
     _test_bitfield_details(flag=2)
 
 def test_bitfield_as_big_endian():
-    if '__pypy__' in sys.builtin_module_names:
-        py.test.skip("no big endian machine supported on pypy for now")
     _test_bitfield_details(flag=4)
 
 
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -139,6 +139,10 @@
             assert rctime.ctime(t) != rctime.asctime(rctime.gmtime(t))
         ltime = rctime.localtime()
         assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
+        try:
+            assert rctime.asctime((12345,) + (0,) * 8).split()[-1] == '12345'
+        except ValueError:
+            pass  # some OS (ie POSIXes besides Linux) reject year > 9999
 
     def test_accept2dyear_access(self):
         import time as rctime
@@ -146,7 +150,8 @@
         accept2dyear = rctime.accept2dyear
         del rctime.accept2dyear
         try:
-            assert rctime.asctime((12345,) + (0,) * 8).split()[-1] == '12345'
+            # with year >= 1900 this shouldn't need to access accept2dyear
+            assert rctime.asctime((2000,) + (0,) * 8).split()[-1] == '2000'
         finally:
             rctime.accept2dyear = accept2dyear
 
diff --git a/pypy/tool/pypyjit.py b/pypy/tool/pypyjit.py
--- a/pypy/tool/pypyjit.py
+++ b/pypy/tool/pypyjit.py
@@ -28,15 +28,16 @@
 config.translation.backendopt.inline_threshold = 0.1
 config.translation.gc = 'boehm'
 config.translating = True
+config.translation.rweakref = False
 set_opt_level(config, level='jit')
 config.objspace.allworkingmodules = False
 config.objspace.usemodules.pypyjit = True
 config.objspace.usemodules.array = False
-config.objspace.usemodules._weakref = True
+config.objspace.usemodules._weakref = False
 config.objspace.usemodules._sre = False
 config.objspace.usemodules._lsprof = False
 #
-config.objspace.usemodules._ffi = True
+config.objspace.usemodules._ffi = False
 config.objspace.usemodules.micronumpy = False
 #
 set_pypy_opt_level(config, level='jit')
@@ -101,7 +102,7 @@
 
     from rpython.jit.codewriter.codewriter import CodeWriter
     CodeWriter.debug = True
-    from rpython.jit.tl.pypyjit_child import run_child
+    from pypy.tool.pypyjit_child import run_child
     run_child(globals(), locals())
 
 
diff --git a/pypy/tool/pypyjit_demo.py b/pypy/tool/pypyjit_demo.py
--- a/pypy/tool/pypyjit_demo.py
+++ b/pypy/tool/pypyjit_demo.py
@@ -1,27 +1,20 @@
-import pypyjit
-pypyjit.set_param(threshold=200)
 
-kwargs = {"z": 1}
+def g(i):
+    k = 0
+    while k < 3:
+        k += 1
+    return i + 1
 
-def f(*args, **kwargs):
-    result = g(1, *args, **kwargs)
-    return result + 2
+def f(x):
+    for i in range(10000):
+        t = (1, 2, i)
+        i = g(i)
+        x == t
 
-def g(x, y, z=2):
-    return x - y + z
-
-def main():
-    res = 0
-    i = 0
-    while i < 10000:
-        res = f(res, z=i)
-        g(1, res, **kwargs)
-        i += 1
-    return res
 
 
 try:
-    print main()
+    f((1, 2, 3))
 
 except Exception, e:
     print "Exception: ", type(e)
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -6,7 +6,7 @@
     def __init__(self):
         self.reset()
 
-    def reset(self):
+    def reset(self, reset_virtuals=True):
         # contains boxes where the class is already known
         self.known_class_boxes = {}
         # store the boxes that contain newly allocated objects, this maps the
@@ -14,7 +14,8 @@
         # escaped the trace or not (True means the box never escaped, False
         # means it did escape), its presences in the mapping shows that it was
         # allocated inside the trace
-        self.new_boxes = {}
+        if reset_virtuals:
+            self.new_boxes = {}
         # Tracks which boxes should be marked as escaped when the key box
         # escapes.
         self.dependencies = {}
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2058,7 +2058,7 @@
                 duplicates[box] = None
 
     def reached_loop_header(self, greenboxes, redboxes, resumedescr):
-        self.heapcache.reset()
+        self.heapcache.reset(reset_virtuals=False)
 
         duplicates = {}
         self.remove_consts_and_duplicates(redboxes, len(redboxes),
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3359,6 +3359,26 @@
         assert res == main(1)
         self.check_resops(call=0, getfield_gc=0)
 
+    def test_isvirtual_call_assembler(self):
+        driver = JitDriver(greens = ['code'], reds = ['n'])
+
+        @look_inside_iff(lambda t1, t2: isvirtual(t1))
+        def g(t1, t2):
+            return t1[0] == t2[0]
+
+        def f(code, n):
+            while n > 0:
+                driver.can_enter_jit(code=code, n=n)
+                driver.jit_merge_point(code=code, n=n)
+                t = (1, 2, n)
+                if code:
+                    f(0, 3)
+                g(t, (1, 2, n))
+                n -= 1
+
+        self.meta_interp(f, [1, 10], inline=True)
+        self.check_resops(call=0, call_may_force=0, call_assembler=2)
+
     def test_reuse_elidable_result(self):
         driver = JitDriver(reds=['n', 's'], greens = [])
         def main(n):

From noreply at buildbot.pypy.org  Wed Oct 23 18:16:24 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 23 Oct 2013 18:16:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix numpypy test_get_include
Message-ID: <20131023161624.1B2F51C0204@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67533:ffeac57b6a3e
Date: 2013-10-23 12:15 -0400
http://bitbucket.org/pypy/pypy/changeset/ffeac57b6a3e/

Log:	fix numpypy test_get_include

diff --git a/lib_pypy/numpypy/lib/utils.py b/lib_pypy/numpypy/lib/utils.py
--- a/lib_pypy/numpypy/lib/utils.py
+++ b/lib_pypy/numpypy/lib/utils.py
@@ -21,14 +21,4 @@
         ...
 
     """
-    try:
-        import numpy
-    except:
-        # running from pypy source directory
-        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-        return os.path.join(head, '../include')
-    else:
-        # using installed numpy core headers
-        import numpy.core as core
-        d = os.path.join(os.path.dirname(core.__file__), 'include')
-    return d
+    return os.path.join(os.path.dirname(__file__), '../../../include')
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -84,6 +84,7 @@
 
     def build_and_convert(self, space, box):
         return self.itemtype.build_and_convert(space, self, box)
+
     def coerce(self, space, w_item):
         return self.itemtype.coerce(space, self, w_item)
 

From noreply at buildbot.pypy.org  Wed Oct 23 18:20:49 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 23 Oct 2013 18:20:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: Also log the function we were trying to
 call which resulted in the vable escape
Message-ID: <20131023162049.8451C1C0204@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67534:fcd59049939d
Date: 2013-10-23 09:19 -0700
http://bitbucket.org/pypy/pypy/changeset/fcd59049939d/

Log:	Also log the function we were trying to call which resulted in the
	vable escape

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1398,7 +1398,7 @@
                     assembler_call_jd)
             if resbox is not None:
                 self.make_result_of_lastop(resbox)
-            self.metainterp.vable_after_residual_call()
+            self.metainterp.vable_after_residual_call(funcbox)
             self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None)
             if vablebox is not None:
                 self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None)
@@ -2437,7 +2437,7 @@
                 # it by ConstPtr(NULL).
                 self.stop_tracking_virtualref(i)
 
-    def vable_after_residual_call(self):
+    def vable_after_residual_call(self, funcbox):
         vinfo = self.jitdriver_sd.virtualizable_info
         if vinfo is not None:
             virtualizable_box = self.virtualizable_boxes[-1]
@@ -2445,7 +2445,14 @@
             if vinfo.tracing_after_residual_call(virtualizable):
                 # the virtualizable escaped during CALL_MAY_FORCE.
                 self.load_fields_from_virtualizable()
-                debug_print('vable escaped during a call in: %s' % self.framestack[-1].jitcode.name)
+                target_name = self.staticdata.get_name_from_address(funcbox.getaddr())
+                if target_name:
+                    target_name = "ConstClass(%s)" % target_name
+                else:
+                    target_name = str(funcbox.getaddr())
+                debug_print('vable escaped during a call in %s to %s' % (
+                    self.framestack[-1].jitcode.name, target_name
+                ))
                 raise SwitchToBlackhole(Counters.ABORT_ESCAPE,
                                         raising_exception=True)
                 # ^^^ we set 'raising_exception' to True because we must still

From noreply at buildbot.pypy.org  Wed Oct 23 18:20:50 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 23 Oct 2013 18:20:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20131023162050.C4DC61C0204@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67535:84ab919963ab
Date: 2013-10-23 09:20 -0700
http://bitbucket.org/pypy/pypy/changeset/84ab919963ab/

Log:	merged upstream

diff --git a/lib_pypy/numpypy/lib/utils.py b/lib_pypy/numpypy/lib/utils.py
--- a/lib_pypy/numpypy/lib/utils.py
+++ b/lib_pypy/numpypy/lib/utils.py
@@ -21,14 +21,4 @@
         ...
 
     """
-    try:
-        import numpy
-    except:
-        # running from pypy source directory
-        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-        return os.path.join(head, '../include')
-    else:
-        # using installed numpy core headers
-        import numpy.core as core
-        d = os.path.join(os.path.dirname(core.__file__), 'include')
-    return d
+    return os.path.join(os.path.dirname(__file__), '../../../include')
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -84,6 +84,7 @@
 
     def build_and_convert(self, space, box):
         return self.itemtype.build_and_convert(space, self, box)
+
     def coerce(self, space, w_item):
         return self.itemtype.coerce(space, self, w_item)
 

From noreply at buildbot.pypy.org  Wed Oct 23 22:00:32 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 23 Oct 2013 22:00:32 +0200 (CEST)
Subject: [pypy-commit] stmgc default: fix wrong assert
Message-ID: <20131023200032.5E43F1C11BF@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r539:d92fcb9e5246
Date: 2013-10-23 22:00 +0200
http://bitbucket.org/pypy/stmgc/changeset/d92fcb9e5246/

Log:	fix wrong assert

diff --git a/c4/steal.c b/c4/steal.c
--- a/c4/steal.c
+++ b/c4/steal.c
@@ -137,7 +137,8 @@
         assert(IMPLIES(!(L->h_tid & GCFLAG_OLD), L->h_original));
         assert(IMPLIES(L->h_tid & GCFLAG_OLD,
                        (B->h_original == (revision_t)L) 
-                       || (B->h_original == L->h_original)));
+                       || (B->h_original == L->h_original)
+                       || (L->h_original == (revision_t)B)));
         if (!L->h_original && L->h_tid & GCFLAG_OLD) {
             /* If old, L must be the original */
             B->h_original = (revision_t)L;

From noreply at buildbot.pypy.org  Wed Oct 23 22:06:18 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 23 Oct 2013 22:06:18 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: import stmgc
Message-ID: <20131023200618.B867B1C3615@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67536:d38e75218673
Date: 2013-10-23 22:04 +0200
http://bitbucket.org/pypy/pypy/changeset/d38e75218673/

Log:	import stmgc

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-2c23968e3d8f
+d92fcb9e5246
diff --git a/rpython/translator/stm/src_stm/steal.c b/rpython/translator/stm/src_stm/steal.c
--- a/rpython/translator/stm/src_stm/steal.c
+++ b/rpython/translator/stm/src_stm/steal.c
@@ -138,7 +138,8 @@
         assert(IMPLIES(!(L->h_tid & GCFLAG_OLD), L->h_original));
         assert(IMPLIES(L->h_tid & GCFLAG_OLD,
                        (B->h_original == (revision_t)L) 
-                       || (B->h_original == L->h_original)));
+                       || (B->h_original == L->h_original)
+                       || (L->h_original == (revision_t)B)));
         if (!L->h_original && L->h_tid & GCFLAG_OLD) {
             /* If old, L must be the original */
             B->h_original = (revision_t)L;

From noreply at buildbot.pypy.org  Wed Oct 23 22:06:20 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 23 Oct 2013 22:06:20 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: fix missing invalidation of read
 status when doing a A2V barrier
Message-ID: <20131023200620.04BDF1C3615@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67537:324cb23503bd
Date: 2013-10-23 22:05 +0200
http://bitbucket.org/pypy/pypy/changeset/324cb23503bd/

Log:	fix missing invalidation of read status when doing a A2V barrier

diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -239,7 +239,7 @@
         if write_barrier_descr is None:
             return v_base    # no barrier needed
 
-        if target_category == 'W':
+        if target_category in ('W', 'V'):
             # if *any* of the readable vars is the same object,
             # it must repeat the read_barrier now
             self.invalidate_read_categories(v_base)

From noreply at buildbot.pypy.org  Wed Oct 23 22:15:24 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 23 Oct 2013 22:15:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: adjust type names for py3
Message-ID: <20131023201524.D07681C2FBD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67538:143ae3044dde
Date: 2013-10-23 13:13 -0700
http://bitbucket.org/pypy/pypy/changeset/143ae3044dde/

Log:	adjust type names for py3

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -680,12 +680,12 @@
     __module__ = "numpypy",
 )
 
-W_StringBox.typedef = TypeDef("string_", (W_CharacterBox.typedef, str_typedef),
+W_StringBox.typedef = TypeDef("bytes_", (W_CharacterBox.typedef, str_typedef),
     __module__ = "numpypy",
     __new__ = interp2app(W_StringBox.descr__new__string_box.im_func),
 )
 
-W_UnicodeBox.typedef = TypeDef("unicode_", (W_CharacterBox.typedef, unicode_typedef),
+W_UnicodeBox.typedef = TypeDef("str_", (W_CharacterBox.typedef, unicode_typedef),
     __module__ = "numpypy",
     __new__ = interp2app(W_UnicodeBox.descr__new__unicode_box.im_func),
 )

From noreply at buildbot.pypy.org  Wed Oct 23 22:15:26 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 23 Oct 2013 22:15:26 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20131023201526.0AD631C2FBD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67539:6965cd292e2b
Date: 2013-10-23 13:14 -0700
http://bitbucket.org/pypy/pypy/changeset/6965cd292e2b/

Log:	2to3

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -3,7 +3,7 @@
 from . import lib
 from .lib import *
 
-from __builtin__ import bool, int, long, float, complex, object, unicode, str
+from builtins import bool, int, int, float, complex, object, str, str
 
 from .core import round, abs, max, min
 
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
--- a/lib_pypy/numpypy/core/__init__.py
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -1,4 +1,4 @@
-from __future__ import division, absolute_import, print_function
+
 
 from . import multiarray
 from . import umath
diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
--- a/lib_pypy/numpypy/core/_methods.py
+++ b/lib_pypy/numpypy/core/_methods.py
@@ -1,9 +1,9 @@
 # Array methods which are called by the both the C-code for the method
 # and the Python code for the NumPy-namespace function
 
-import multiarray as mu
-import umath as um
-from numeric import asanyarray
+from . import multiarray as mu
+from . import umath as um
+from .numeric import asanyarray
 
 def _amax(a, axis=None, out=None, keepdims=False):
     return um.maximum.reduce(a, axis=axis,
@@ -31,7 +31,7 @@
 
 def _count_reduce_items(arr, axis):
     if axis is None:
-        axis = tuple(xrange(arr.ndim))
+        axis = tuple(range(arr.ndim))
     if not isinstance(axis, tuple):
         axis = (axis,)
     items = 1
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ b/lib_pypy/numpypy/core/arrayprint.py
@@ -13,10 +13,10 @@
 # and by Travis Oliphant  2005-8-22 for numpy
 
 import sys
-import numerictypes as _nt
-from umath import maximum, minimum, absolute, not_equal, isnan, isinf
+from . import numerictypes as _nt
+from .umath import maximum, minimum, absolute, not_equal, isnan, isinf
 #from multiarray import format_longfloat, datetime_as_string, datetime_data
-from fromnumeric import ravel
+from .fromnumeric import ravel
 
 
 def product(x, y): return x*y
@@ -194,7 +194,7 @@
     return d
 
 def _leading_trailing(a):
-    import numeric as _nc
+    from . import numeric as _nc
     if a.ndim == 1:
         if len(a) > 2*_summaryEdgeItems:
             b = _nc.concatenate((a[:_summaryEdgeItems],
@@ -258,9 +258,9 @@
                   'str' : str}
 
     if formatter is not None:
-        fkeys = [k for k in formatter.keys() if formatter[k] is not None]
+        fkeys = [k for k in list(formatter.keys()) if formatter[k] is not None]
         if 'all' in fkeys:
-            for key in formatdict.keys():
+            for key in list(formatdict.keys()):
                 formatdict[key] = formatter['all']
         if 'int_kind' in fkeys:
             for key in ['int']:
@@ -274,7 +274,7 @@
         if 'str_kind' in fkeys:
             for key in ['numpystr', 'str']:
                 formatdict[key] = formatter['str_kind']
-        for key in formatdict.keys():
+        for key in list(formatdict.keys()):
             if key in fkeys:
                 formatdict[key] = formatter[key]
 
@@ -322,7 +322,7 @@
     return lst
 
 def _convert_arrays(obj):
-    import numeric as _nc
+    from . import numeric as _nc
     newtup = []
     for k in obj:
         if isinstance(k, _nc.ndarray):
@@ -478,14 +478,14 @@
     if rank == 1:
         s = ""
         line = next_line_prefix
-        for i in xrange(leading_items):
+        for i in range(leading_items):
             word = format_function(a[i]) + separator
             s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
 
         if summary_insert1:
             s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix)
 
-        for i in xrange(trailing_items, 1, -1):
+        for i in range(trailing_items, 1, -1):
             word = format_function(a[-i]) + separator
             s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
 
@@ -496,7 +496,7 @@
     else:
         s = '['
         sep = separator.rstrip()
-        for i in xrange(leading_items):
+        for i in range(leading_items):
             if i > 0:
                 s += next_line_prefix
             s += _formatArray(a[i], format_function, rank-1, max_line_len,
@@ -507,7 +507,7 @@
         if summary_insert1:
             s += next_line_prefix + summary_insert1 + "\n"
 
-        for i in xrange(trailing_items, 1, -1):
+        for i in range(trailing_items, 1, -1):
             if leading_items or i != trailing_items:
                 s += next_line_prefix
             s += _formatArray(a[-i], format_function, rank-1, max_line_len,
@@ -537,7 +537,7 @@
             pass
 
     def fillFormat(self, data):
-        import numeric as _nc
+        from . import numeric as _nc
         errstate = _nc.seterr(all='ignore')
         try:
             special = isnan(data) | isinf(data)
@@ -590,7 +590,7 @@
         self.format = format
 
     def __call__(self, x, strip_zeros=True):
-        import numeric as _nc
+        from . import numeric as _nc
         err = _nc.seterr(invalid='ignore')
         try:
             if isnan(x):
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -4,7 +4,7 @@
 """Module containing non-deprecated functions borrowed from Numeric.
 
 """
-from __future__ import division, absolute_import, print_function
+
 
 import types
 
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -1,4 +1,4 @@
-from __future__ import division, absolute_import, print_function
+
 
 __all__ = [
            'newaxis', 'ufunc', 'argwhere',
@@ -27,7 +27,7 @@
     try:
         mall = getattr(module, '__all__')
     except AttributeError:
-        mall = [k for k in module.__dict__.keys() if not k.startswith('_')]
+        mall = [k for k in list(module.__dict__.keys()) if not k.startswith('_')]
     for a in mall:
         if a not in adict:
             __all__.append(a)
diff --git a/lib_pypy/numpypy/core/numerictypes.py b/lib_pypy/numpypy/core/numerictypes.py
--- a/lib_pypy/numpypy/core/numerictypes.py
+++ b/lib_pypy/numpypy/core/numerictypes.py
@@ -83,7 +83,7 @@
      \\-> object_ (not used much)                (kind=O)
 
 """
-from __future__ import division, absolute_import, print_function
+
 
 # we add more at the bottom
 __all__ = ['sctypeDict', 'sctypeNA', 'typeDict', 'typeNA', 'sctypes',
@@ -102,9 +102,9 @@
 # as numerictypes.bool, etc.
 if sys.version_info[0] >= 3:
     from builtins import bool, int, float, complex, object, str
-    unicode = str
+    str = str
 else:
-    from __builtin__ import bool, int, float, complex, object, unicode, str
+    from builtins import bool, int, float, complex, object, str, str
 
 
 # String-handling utilities to avoid locale-dependence.
@@ -285,7 +285,7 @@
 
 
 def _add_types():
-    for a in typeinfo.keys():
+    for a in list(typeinfo.keys()):
         name = english_lower(a)
         if isinstance(typeinfo[a], tuple):
             typeobj = typeinfo[a][-1]
@@ -301,7 +301,7 @@
 _add_types()
 
 def _add_aliases():
-    for a in typeinfo.keys():
+    for a in list(typeinfo.keys()):
         name = english_lower(a)
         if not isinstance(typeinfo[a], tuple):
             continue
@@ -312,7 +312,7 @@
         if base != '':
             myname = "%s%d" % (base, bit)
             if (name != 'longdouble' and name != 'clongdouble') or \
-                   myname not in allTypes.keys():
+                   myname not in list(allTypes.keys()):
                 allTypes[myname] = typeobj
                 sctypeDict[myname] = typeobj
                 if base == 'complex':
@@ -354,7 +354,7 @@
         uval = typeinfo['U'+ctype]
         typeobj = val[-1]
         utypeobj = uval[-1]
-        if intname not in allTypes.keys():
+        if intname not in list(allTypes.keys()):
             uintname = 'uint%d' % bits
             allTypes[intname] = typeobj
             allTypes[uintname] = utypeobj
@@ -432,7 +432,7 @@
 # Now, construct dictionary to lookup character codes from types
 _sctype2char_dict = {}
 def _construct_char_code_lookup():
-    for name in typeinfo.keys():
+    for name in list(typeinfo.keys()):
         tup = typeinfo[name]
         if isinstance(tup, tuple):
             if tup[0] not in ['p', 'P']:
@@ -444,7 +444,7 @@
            'uint':[],
            'float':[],
            'complex':[],
-           'others':[bool, object, str, unicode, void]}
+           'others':[bool, object, str, str, void]}
 
 def _add_array_type(typename, bits):
     try:
@@ -545,7 +545,7 @@
                  complex: 'complex_',
                  bool: 'bool_',
                  bytes: 'bytes_',
-                 unicode: 'unicode_',
+                 str: 'unicode_',
                  buffer_type: 'void',
                 }
 
@@ -784,7 +784,7 @@
 _maxvals = _typedict()
 _minvals = _typedict()
 def _construct_lookups():
-    for name, val in typeinfo.items():
+    for name, val in list(typeinfo.items()):
         if not isinstance(val, tuple):
             continue
         obj = val[-1]
@@ -859,21 +859,21 @@
     # Py3K
     ScalarType = [int, float, complex, int, bool, bytes, str, memoryview]
 
-ScalarType.extend(_sctype2char_dict.keys())
+ScalarType.extend(list(_sctype2char_dict.keys()))
 ScalarType = tuple(ScalarType)
-for key in _sctype2char_dict.keys():
+for key in list(_sctype2char_dict.keys()):
     cast[key] = lambda x, k=key : array(x, copy=False).astype(k)
 
 # Create the typestring lookup dictionary
 _typestr = _typedict()
-for key in _sctype2char_dict.keys():
+for key in list(_sctype2char_dict.keys()):
     if issubclass(key, allTypes['flexible']):
         _typestr[key] = _sctype2char_dict[key]
     else:
         _typestr[key] = empty((1,), key).dtype.str[1:]
 
 # Make sure all typestrings are in sctypeDict
-for key, val in _typestr.items():
+for key, val in list(_typestr.items()):
     if val not in sctypeDict:
         sctypeDict[val] = key
 
diff --git a/lib_pypy/numpypy/core/shape_base.py b/lib_pypy/numpypy/core/shape_base.py
--- a/lib_pypy/numpypy/core/shape_base.py
+++ b/lib_pypy/numpypy/core/shape_base.py
@@ -1,4 +1,4 @@
-from __future__ import division, absolute_import, print_function
+
 
 __all__ = ['atleast_1d', 'atleast_2d', 'atleast_3d', 'vstack', 'hstack']
 
diff --git a/lib_pypy/numpypy/lib/__init__.py b/lib_pypy/numpypy/lib/__init__.py
--- a/lib_pypy/numpypy/lib/__init__.py
+++ b/lib_pypy/numpypy/lib/__init__.py
@@ -1,4 +1,4 @@
-from __future__ import division, absolute_import, print_function
+
 
 import math
 
diff --git a/lib_pypy/numpypy/lib/shape_base.py b/lib_pypy/numpypy/lib/shape_base.py
--- a/lib_pypy/numpypy/lib/shape_base.py
+++ b/lib_pypy/numpypy/lib/shape_base.py
@@ -51,4 +51,4 @@
            [[3, 4]]])
 
     """
-    return _nx.concatenate(map(atleast_3d,tup),2)
+    return _nx.concatenate(list(map(atleast_3d,tup)),2)
diff --git a/lib_pypy/numpypy/lib/ufunclike.py b/lib_pypy/numpypy/lib/ufunclike.py
--- a/lib_pypy/numpypy/lib/ufunclike.py
+++ b/lib_pypy/numpypy/lib/ufunclike.py
@@ -3,7 +3,7 @@
 storing results in an output array.
 
 """
-from __future__ import division, absolute_import, print_function
+
 
 __all__ = ['fix', 'isneginf', 'isposinf']
 

From noreply at buildbot.pypy.org  Wed Oct 23 22:15:27 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 23 Oct 2013 22:15:27 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: kill
Message-ID: <20131023201527.2B8601C2FBD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67540:5acd4d6e2c25
Date: 2013-10-23 13:14 -0700
http://bitbucket.org/pypy/pypy/changeset/5acd4d6e2c25/

Log:	kill

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -34,7 +34,6 @@
         'path'                  : 'state.get(space).w_path',
         'modules'               : 'state.get(space).w_modules', 
         'argv'                  : 'state.get(space).w_argv',
-        'py3kwarning'           : 'space.w_False',
         'warnoptions'           : 'state.get(space).w_warnoptions', 
         'abiflags'              : 'space.wrap("")',
         'builtin_module_names'  : 'space.w_None',

From noreply at buildbot.pypy.org  Thu Oct 24 02:25:59 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 24 Oct 2013 02:25:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test_rctime on win32 also
Message-ID: <20131024002559.EEEB61C11BF@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67541:48c6b3ff4a7b
Date: 2013-10-23 20:21 -0400
http://bitbucket.org/pypy/pypy/changeset/48c6b3ff4a7b/

Log:	fix test_rctime on win32 also

diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -140,7 +140,7 @@
         ltime = rctime.localtime()
         assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
         try:
-            assert rctime.asctime((12345,) + (0,) * 8).split()[-1] == '12345'
+            rctime.asctime((12345,) + (0,) * 8)  # assert this doesn't crash
         except ValueError:
             pass  # some OS (ie POSIXes besides Linux) reject year > 9999
 

From noreply at buildbot.pypy.org  Thu Oct 24 03:05:58 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 24 Oct 2013 03:05:58 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: Random hacks to make the JIT
 compile things when you have sys.settrace enabled
Message-ID: <20131024010558.090391C2FBD@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67542:226c15c4600b
Date: 2013-10-23 18:05 -0700
http://bitbucket.org/pypy/pypy/changeset/226c15c4600b/

Log:	Random hacks to make the JIT compile things when you have
	sys.settrace enabled

diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -2,6 +2,8 @@
 This module defines the abstract base classes that support execution:
 Code and Frame.
 """
+from rpython.rlib import jit
+
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.baseobjspace import W_Root
 
@@ -85,28 +87,15 @@
         self.w_locals = w_locals
         self.locals2fast()
 
-    def getfastscope(self):
-        "Abstract. Get the fast locals as a list."
-        raise TypeError("abstract")
-
-    def setfastscope(self, scope_w):
-        """Abstract. Initialize the fast locals from a list of values,
-        where the order is according to self.getcode().signature()."""
-        raise TypeError("abstract")
-
-    def getfastscopelength(self):
-        "Abstract. Get the expected number of locals."
-        raise TypeError("abstract")
-
+    @jit.look_inside_iff(lambda self: jit.isvirtual(self))
     def fast2locals(self):
         # Copy values from the fastlocals to self.w_locals
         if self.w_locals is None:
             self.w_locals = self.space.newdict()
         varnames = self.getcode().getvarnames()
-        fastscope_w = self.getfastscope()
-        for i in range(min(len(varnames), self.getfastscopelength())):
+        for i in range(min(len(varnames), self.pycode.co_nlocals)):
             name = varnames[i]
-            w_value = fastscope_w[i]
+            w_value = self.locals_stack_w[i]
             w_name = self.space.wrap(name)
             if w_value is not None:
                 self.space.setitem(self.w_locals, w_name, w_value)
@@ -117,11 +106,12 @@
                     if not e.match(self.space, self.space.w_KeyError):
                         raise
 
+    @jit.look_inside_iff(lambda self: jit.isvirtual(self))
     def locals2fast(self):
         # Copy values from self.w_locals to the fastlocals
         assert self.w_locals is not None
         varnames = self.getcode().getvarnames()
-        numlocals = self.getfastscopelength()
+        numlocals = self.pycode.co_nlocals
 
         new_fastlocals_w = [None] * numlocals
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -76,7 +76,7 @@
                 frame_vref()
             jit.virtual_ref_finish(frame_vref, frame)
 
-        if self.w_tracefunc is not None and not frame.hide():
+        if self.gettrace() is not None and not frame.hide():
             self.space.frame_trace_action.fire()
 
     # ________________________________________________________________
@@ -115,14 +115,14 @@
 
     def call_trace(self, frame):
         "Trace the call of a function"
-        if self.w_tracefunc is not None or self.profilefunc is not None:
+        if self.gettrace() is not None or self.profilefunc is not None:
             self._trace(frame, 'call', self.space.w_None)
             if self.profilefunc:
                 frame.is_being_profiled = True
 
     def return_trace(self, frame, w_retval):
         "Trace the return from a function"
-        if self.w_tracefunc is not None:
+        if self.gettrace() is not None:
             return_from_hidden = self._trace(frame, 'return', w_retval)
             # special case: if we are returning from a hidden function,
             # then maybe we have to fire() the action again; otherwise
@@ -152,7 +152,7 @@
     def exception_trace(self, frame, operationerr):
         "Trace function called upon OperationError."
         operationerr.record_interpreter_traceback()
-        if self.w_tracefunc is not None:
+        if self.gettrace() is not None:
             self._trace(frame, 'exception', None, operationerr)
         #operationerr.print_detailed_traceback(self.space)
 
@@ -181,7 +181,7 @@
             self.space.frame_trace_action.fire()
 
     def gettrace(self):
-        return self.w_tracefunc
+        return jit.promote(self.w_tracefunc)
 
     def setprofile(self, w_func):
         """Set the global trace function."""
@@ -234,7 +234,7 @@
 
         # Tracing cases
         if event == 'call':
-            w_callback = self.w_tracefunc
+            w_callback = self.gettrace()
         else:
             w_callback = frame.w_f_trace
 
@@ -367,7 +367,7 @@
     def _rebuild_action_dispatcher(self):
         periodic_actions = unrolling_iterable(self._periodic_actions)
 
-        @jit.dont_look_inside
+        @jit.unroll_safe
         def action_dispatcher(ec, frame):
             # periodic actions (first reset the bytecode counter)
             self.reset_ticker(self.checkinterval_scaled)
@@ -454,6 +454,9 @@
     def perform(self, executioncontext, frame):
         if self.finalizers_lock_count > 0:
             return
+        self._run_finalizers()
+
+    def _run_finalizers(self):
         # Each call to perform() first grabs the self.dying_objects
         # and replaces it with an empty list.  We do this to try to
         # avoid too deep recursions of the kind of __del__ being called
@@ -473,9 +476,10 @@
 class FrameTraceAction(AsyncAction):
     """An action that calls the local trace functions (w_f_trace)."""
 
+    @jit.unroll_safe
     def perform(self, executioncontext, frame):
         if (frame.w_f_trace is None or executioncontext.is_tracing or
-            executioncontext.w_tracefunc is None):
+            executioncontext.gettrace() is None):
             return
         code = frame.pycode
         if frame.instr_lb <= frame.last_instr < frame.instr_ub:
diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -128,7 +128,7 @@
         if self.cells is not None:
             self.cells[:ncellvars] = cellvars
 
-    @jit.dont_look_inside
+    @jit.look_inside_iff(lambda self: jit.isvirtual(self))
     def fast2locals(self):
         super_fast2locals(self)
         # cellvars are values exported to inner scopes
@@ -147,7 +147,7 @@
                 w_name = self.space.wrap(name)
                 self.space.setitem(self.w_locals, w_name, w_value)
 
-    @jit.dont_look_inside
+    @jit.look_inside_iff(lambda self: jit.isvirtual(self))
     def locals2fast(self):
         super_locals2fast(self)
         freevarnames = self.pycode.co_cellvars + self.pycode.co_freevars
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -440,12 +440,7 @@
     def getcode(self):
         return hint(self.pycode, promote=True)
 
-    @jit.dont_look_inside
-    def getfastscope(self):
-        "Get the fast locals as a list."
-        return self.locals_stack_w
-
-    @jit.dont_look_inside
+    @jit.look_inside_iff(lambda self, scope_w: jit.isvirtual(scope_w))
     def setfastscope(self, scope_w):
         """Initialize the fast locals from a list of values,
         where the order is according to self.pycode.signature()."""
@@ -463,9 +458,6 @@
         This is overridden in nestedscope.py"""
         pass
 
-    def getfastscopelength(self):
-        return self.pycode.co_nlocals
-
     def getclosure(self):
         return None
 
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -75,11 +75,15 @@
                 # and there is a signal pending: we force the ticker to
                 # -1, which should ensure perform() is called quickly.
 
+    def perform(self, executioncontext, frame):
+        self._poll_for_signals()
+
     @jit.dont_look_inside
-    def perform(self, executioncontext, frame):
+    def _poll_for_signals(self):
         # Poll for the next signal, if any
         n = self.pending_signal
-        if n < 0: n = pypysig_poll()
+        if n < 0:
+            n = pypysig_poll()
         while n >= 0:
             if self.space.threadlocals.signals_enabled():
                 # If we are in the main thread, report the signal now,
@@ -87,7 +91,8 @@
                 self.pending_signal = -1
                 report_signal(self.space, n)
                 n = self.pending_signal
-                if n < 0: n = pypysig_poll()
+                if n < 0:
+                    n = pypysig_poll()
             else:
                 # Otherwise, arrange for perform() to be called again
                 # after we switch to the main thread.

From noreply at buildbot.pypy.org  Thu Oct 24 05:33:59 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 24 Oct 2013 05:33:59 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: random cleanup
Message-ID: <20131024033359.55F2C1C3253@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67543:97aaec657093
Date: 2013-10-23 20:33 -0700
http://bitbucket.org/pypy/pypy/changeset/97aaec657093/

Log:	random cleanup

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -189,9 +189,8 @@
         # speed hack
         fresh_frame = jit.hint(frame, access_directly=True,
                                       fresh_virtualizable=True)
-        args_matched = args.parse_into_scope(None, fresh_frame.locals_stack_w,
-                                             func.name,
-                                             sig, func.defs_w)
+        args.parse_into_scope(None, fresh_frame.locals_stack_w, func.name,
+                              sig, func.defs_w)
         fresh_frame.init_cells()
         return frame.run()
 
@@ -202,9 +201,8 @@
         # speed hack
         fresh_frame = jit.hint(frame, access_directly=True,
                                       fresh_virtualizable=True)
-        args_matched = args.parse_into_scope(w_obj, fresh_frame.locals_stack_w,
-                                             func.name,
-                                             sig, func.defs_w)
+        args.parse_into_scope(w_obj, fresh_frame.locals_stack_w, func.name,
+                              sig, func.defs_w)
         fresh_frame.init_cells()
         return frame.run()
 

From noreply at buildbot.pypy.org  Thu Oct 24 08:23:43 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 08:23:43 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: test and fix
Message-ID: <20131024062343.9532D1C135D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67544:685c42b52646
Date: 2013-10-24 08:22 +0200
http://bitbucket.org/pypy/pypy/changeset/685c42b52646/

Log:	test and fix

diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -49,6 +49,12 @@
     def is_double(self):
         return False
 
+    def aslist_int(self, w_ob):
+        return None
+
+    def aslist_float(self, w_ob):
+        return None
+
     def newp(self, w_init):
         space = self.space
         raise operationerrfmt(space.w_TypeError,
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -34,6 +34,15 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
+    def test_bug(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        five = _cffi_backend.cast(LONG, 5)
+        raises(TypeError, list, five)
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
+        raises(TypeError, list, five_and_a_half)
+
 
 class AppTest_fast_path_to_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))

From noreply at buildbot.pypy.org  Thu Oct 24 08:24:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 08:24:08 +0200 (CEST)
Subject: [pypy-commit] pypy array-overallocation-in-nursery: Backed out
 changeset 685c42b52646
Message-ID: <20131024062408.C40211C135D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: array-overallocation-in-nursery
Changeset: r67545:8d7795668fa8
Date: 2013-10-24 08:23 +0200
http://bitbucket.org/pypy/pypy/changeset/8d7795668fa8/

Log:	Backed out changeset 685c42b52646

diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -49,12 +49,6 @@
     def is_double(self):
         return False
 
-    def aslist_int(self, w_ob):
-        return None
-
-    def aslist_float(self, w_ob):
-        return None
-
     def newp(self, w_init):
         space = self.space
         raise operationerrfmt(space.w_TypeError,
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -34,15 +34,6 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
-    def test_bug(self):
-        import _cffi_backend
-        LONG = _cffi_backend.new_primitive_type('long')
-        five = _cffi_backend.cast(LONG, 5)
-        raises(TypeError, list, five)
-        DOUBLE = _cffi_backend.new_primitive_type('double')
-        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
-        raises(TypeError, list, five_and_a_half)
-
 
 class AppTest_fast_path_to_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))

From noreply at buildbot.pypy.org  Thu Oct 24 08:25:53 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 08:25:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: test and fix
Message-ID: <20131024062553.E6F2E1C135D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67546:deb9af94dcc2
Date: 2013-10-24 08:25 +0200
http://bitbucket.org/pypy/pypy/changeset/deb9af94dcc2/

Log:	test and fix

diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -49,6 +49,12 @@
     def is_double(self):
         return False
 
+    def aslist_int(self, w_ob):
+        return None
+
+    def aslist_float(self, w_ob):
+        return None
+
     def newp(self, w_init):
         space = self.space
         raise operationerrfmt(space.w_TypeError,
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -34,6 +34,15 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
+    def test_bug(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        five = _cffi_backend.cast(LONG, 5)
+        raises(TypeError, list, five)
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
+        raises(TypeError, list, five_and_a_half)
+
 
 class AppTest_fast_path_to_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))

From noreply at buildbot.pypy.org  Thu Oct 24 09:04:28 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 09:04:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move the test to the right class.
Message-ID: <20131024070428.83B071C135D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67547:ec8875093ab9
Date: 2013-10-24 08:52 +0200
http://bitbucket.org/pypy/pypy/changeset/ec8875093ab9/

Log:	Move the test to the right class.

diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -34,15 +34,6 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
-    def test_bug(self):
-        import _cffi_backend
-        LONG = _cffi_backend.new_primitive_type('long')
-        five = _cffi_backend.cast(LONG, 5)
-        raises(TypeError, list, five)
-        DOUBLE = _cffi_backend.new_primitive_type('double')
-        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
-        raises(TypeError, list, five_and_a_half)
-
 
 class AppTest_fast_path_to_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
@@ -93,6 +84,14 @@
         pbuf = _cffi_backend.cast(P_LONG, buf)
         raises(TypeError, "list(pbuf)")
 
+    def test_bug(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        five = _cffi_backend.cast(LONG, 5)
+        raises(TypeError, list, five)
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
+        raises(TypeError, list, five_and_a_half)
 
     def test_list_float(self):
         import _cffi_backend

From noreply at buildbot.pypy.org  Thu Oct 24 09:04:29 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 09:04:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix
Message-ID: <20131024070429.B65A61C135D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67548:be1986c9f303
Date: 2013-10-24 09:03 +0200
http://bitbucket.org/pypy/pypy/changeset/be1986c9f303/

Log:	Test and fix

diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -95,14 +95,13 @@
 
     def convert_array_from_object(self, cdata, w_ob):
         space = self.space
-        if self._convert_array_from_list_strategy_maybe(cdata, w_ob):
-            # the fast path worked, we are done now
-            return
-        #
-        # continue with the slow path
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
-            self._convert_array_from_listview(cdata, w_ob)
+            #
+            if not self._convert_array_from_list_strategy_maybe(cdata, w_ob):
+                # continue with the slow path
+                self._convert_array_from_listview(cdata, w_ob)
+            #
         elif (self.can_cast_anything or
               (self.ctitem.is_primitive_integer and
                self.ctitem.size == rffi.sizeof(lltype.Char))):
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -35,6 +35,19 @@
         assert buf[2] == 3.3
 
 
+class AppTest_fast_path_bug(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def test_bug_not_list_or_tuple(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY_2 = _cffi_backend.new_array_type(P_LONG, 2)
+        P_LONG_ARRAY_2 = _cffi_backend.new_pointer_type(LONG_ARRAY_2)
+        LONG_ARRAY_ARRAY = _cffi_backend.new_array_type(P_LONG_ARRAY_2, None)
+        raises(TypeError, _cffi_backend.newp, LONG_ARRAY_ARRAY, [set([4, 5])])
+
+
 class AppTest_fast_path_to_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
 

From noreply at buildbot.pypy.org  Thu Oct 24 09:32:19 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 09:32:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move code around
Message-ID: <20131024073219.338FA1C30A6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67549:33365ef75d96
Date: 2013-10-24 09:19 +0200
http://bitbucket.org/pypy/pypy/changeset/33365ef75d96/

Log:	Move code around

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -283,10 +283,18 @@
         return self.ctype.iter(self)
 
     def unpackiterable_int(self, space):
-        return self.ctype.aslist_int(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_int_items(self)
+        return None
 
     def unpackiterable_float(self, space):
-        return self.ctype.aslist_float(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_float_items(self)
+        return None
 
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -105,26 +105,6 @@
     def iter(self, cdata):
         return W_CDataIter(self.space, self.ctitem, cdata)
 
-    def aslist_int(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_long():
-            res = []
-            buf = rffi.cast(rffi.LONGP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
-    def aslist_float(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_double():
-            res = []
-            buf = rffi.cast(rffi.DOUBLEP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
     def get_vararg_type(self):
         return self.ctptr
 
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -43,18 +43,15 @@
     def is_unichar_ptr_or_array(self):
         return False
 
-    def is_long(self):
-        return False
-
-    def is_double(self):
-        return False
-
-    def aslist_int(self, w_ob):
+    def unpack_list_of_int_items(self, cdata):
         return None
 
-    def aslist_float(self, w_ob):
+    def unpack_list_of_float_items(self, cdata):
         return None
 
+    def pack_list_of_items(self, cdata, w_ob):
+        return False
+
     def newp(self, w_init):
         space = self.space
         raise operationerrfmt(space.w_TypeError,
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -170,9 +170,6 @@
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
 
-    def is_long(self):
-        return self.size == rffi.sizeof(lltype.Signed)
-
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
 
@@ -204,6 +201,26 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_signed_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.LONG): # XXX
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        if self.size == rffi.sizeof(rffi.LONG): # XXX
+            int_list = self.space.listview_int(w_ob)
+            if int_list is not None:
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.LONGP, cdata)
+                copy_list_to_raw_array(int_list, cdata)
+                return True
+        return False
+
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
     _attrs_            = ['value_fits_long', 'value_fits_ulong', 'vrangemax']
@@ -276,9 +293,6 @@
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
 
-    def is_double(self):
-        return self.size == rffi.sizeof(lltype.Float)
-
     def cast(self, w_ob):
         space = self.space
         if isinstance(w_ob, cdataobj.W_CData):
@@ -318,6 +332,26 @@
         value = space.float_w(space.float(w_ob))
         misc.write_raw_float_data(cdata, value, self.size)
 
+    def unpack_list_of_float_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.DOUBLE): # XXX
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        if self.size == rffi.sizeof(rffi.DOUBLE): # XXX
+            float_list = self.space.listview_float(w_ob)
+            if float_list is not None:
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.DOUBLEP, cdata)
+                copy_list_to_raw_array(float_list, cdata)
+                return True
+        return False
+
 
 class W_CTypePrimitiveLongDouble(W_CTypePrimitiveFloat):
     _attrs_ = []
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -42,12 +42,6 @@
     def is_char_or_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveCharOrUniChar)
 
-    def aslist_int(self, cdata):
-        return None
-
-    def aslist_float(self, cdata):
-        return None
-
     def cast(self, w_ob):
         # cast to a pointer, to a funcptr, or to an array.
         # Note that casting to an array is an extension to the C language,
@@ -64,24 +58,10 @@
             value = rffi.cast(rffi.CCHARP, value)
         return cdataobj.W_CData(space, value, self)
 
-    def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
-        from rpython.rlib.rarray import copy_list_to_raw_array
-        int_list = self.space.listview_int(w_ob)
-        float_list = self.space.listview_float(w_ob)
+    def _convert_array_from_listview(self, cdata, w_ob):
+        if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
+            return
         #
-        if self.ctitem.is_long() and int_list is not None:
-            cdata = rffi.cast(rffi.LONGP, cdata)
-            copy_list_to_raw_array(int_list, cdata)
-            return True
-        #
-        if self.ctitem.is_double() and float_list is not None:
-            cdata = rffi.cast(rffi.DOUBLEP, cdata)
-            copy_list_to_raw_array(float_list, cdata)
-            return True
-        #
-        return False
-
-    def _convert_array_from_listview(self, cdata, w_ob):
         space = self.space
         lst_w = space.listview(w_ob)
         if self.length >= 0 and len(lst_w) > self.length:
@@ -97,11 +77,7 @@
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
-            #
-            if not self._convert_array_from_list_strategy_maybe(cdata, w_ob):
-                # continue with the slow path
-                self._convert_array_from_listview(cdata, w_ob)
-            #
+            self._convert_array_from_listview(cdata, w_ob)
         elif (self.can_cast_anything or
               (self.ctitem.is_primitive_integer and
                self.ctitem.size == rffi.sizeof(lltype.Char))):
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -1,18 +1,18 @@
-# side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule()
+# side-effect: FORMAT_LONGDOUBLE must be built before the first test
 from pypy.module._cffi_backend import misc
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+
 
 class AppTest_fast_path_from_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
 
     def setup_method(self, meth):
-        def forbidden(self, *args):
+        def forbidden(*args):
             assert False, 'The slow path is forbidden'
-        self._original = W_CTypePtrOrArray._convert_array_from_listview.im_func
-        W_CTypePtrOrArray._convert_array_from_listview = forbidden
+        self._original = self.space.listview
+        self.space.listview = forbidden
 
     def teardown_method(self, meth):
-        W_CTypePtrOrArray._convert_array_from_listview = self._original
+        self.space.listview = self._original
 
     def test_fast_init_from_list(self):
         import _cffi_backend

From noreply at buildbot.pypy.org  Thu Oct 24 10:03:45 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 10:03:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fast packing for arrays of other
 integer types than 'long'.
Message-ID: <20131024080345.BD06B1C0204@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67550:beb446933cde
Date: 2013-10-24 09:54 +0200
http://bitbucket.org/pypy/pypy/changeset/beb446933cde/

Log:	Fast packing for arrays of other integer types than 'long'.

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -2,6 +2,7 @@
 Primitives.
 """
 
+import sys
 from pypy.interpreter.error import operationerrfmt
 
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
@@ -212,14 +213,25 @@
         return None
 
     def pack_list_of_items(self, cdata, w_ob):
-        if self.size == rffi.sizeof(rffi.LONG): # XXX
-            int_list = self.space.listview_int(w_ob)
-            if int_list is not None:
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            if self.size == rffi.sizeof(rffi.LONG): # fastest path
                 from rpython.rlib.rarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
-                return True
-        return False
+            else:
+                if self.value_fits_long:
+                    vmin = self.vmin
+                    vrangemax = self.vrangemax
+                else:
+                    vmin = r_uint(0)
+                    vrangemax = r_uint(-1)
+                overflowed = misc.pack_list_to_raw_array_bounds(
+                    int_list, cdata, self.size, vmin, vrangemax)
+                if overflowed != 0:
+                    self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
 
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
@@ -272,6 +284,20 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_unsigned_data(value)
 
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            if self.value_fits_long:
+                vrangemax = self.vrangemax
+            else:
+                vrangemax = r_uint(sys.maxint)
+            overflowed = misc.pack_list_to_raw_array_bounds(
+                int_list, cdata, self.size, r_uint(0), vrangemax)
+            if overflowed != 0:
+                self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
     _attrs_ = []
@@ -350,7 +376,7 @@
                 cdata = rffi.cast(rffi.DOUBLEP, cdata)
                 copy_list_to_raw_array(float_list, cdata)
                 return True
-        return False
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
 
 
 class W_CTypePrimitiveLongDouble(W_CTypePrimitiveFloat):
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -315,3 +315,17 @@
             _raw_memclear_tp(TP, TPP, dest)
             return
     raise NotImplementedError("bad clear size")
+
+# ____________________________________________________________
+
+def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) - vmin > vrangemax:
+                    return x      # overflow
+                ptr[i] = rffi.cast(TP, x)
+            return 0
+    raise NotImplementedError("bad integer size")
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -1,5 +1,6 @@
 # side-effect: FORMAT_LONGDOUBLE must be built before the first test
 from pypy.module._cffi_backend import misc
+from pypy.module._cffi_backend.ctypeobj import W_CType
 
 
 class AppTest_fast_path_from_list(object):
@@ -8,11 +9,11 @@
     def setup_method(self, meth):
         def forbidden(*args):
             assert False, 'The slow path is forbidden'
-        self._original = self.space.listview
-        self.space.listview = forbidden
+        self._original = W_CType.pack_list_of_items.im_func
+        W_CType.pack_list_of_items = forbidden
 
     def teardown_method(self, meth):
-        self.space.listview = self._original
+        W_CType.pack_list_of_items = self._original
 
     def test_fast_init_from_list(self):
         import _cffi_backend
@@ -34,6 +35,58 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
+    def test_fast_init_short_from_list(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, None)
+        buf = _cffi_backend.newp(SHORT_ARRAY, [1, -2, 3])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [40000])
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [-40000])
+
+    def test_fast_init_longlong_from_list(self):
+        if type(2 ** 50) is long:
+            large_int = 2 ** 30
+        else:
+            large_int = 2 ** 50
+        import _cffi_backend
+        LONGLONG = _cffi_backend.new_primitive_type('long long')
+        P_LONGLONG = _cffi_backend.new_pointer_type(LONGLONG)
+        LONGLONG_ARRAY = _cffi_backend.new_array_type(P_LONGLONG, None)
+        buf = _cffi_backend.newp(LONGLONG_ARRAY, [1, -2, 3, large_int])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        assert buf[3] == large_int
+
+    def test_fast_init_ushort_from_list(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, None)
+        buf = _cffi_backend.newp(USHORT_ARRAY, [1, 2, 40000])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == 40000
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [70000])
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [-1])
+
+    def test_fast_init_ulong_from_list(self):
+        import sys
+        import _cffi_backend
+        ULONG = _cffi_backend.new_primitive_type('unsigned long')
+        P_ULONG = _cffi_backend.new_pointer_type(ULONG)
+        ULONG_ARRAY = _cffi_backend.new_array_type(P_ULONG, None)
+        buf = _cffi_backend.newp(ULONG_ARRAY, [1, 2, sys.maxint])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == sys.maxint
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-1])
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-sys.maxint])
+
 
 class AppTest_fast_path_bug(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))

From noreply at buildbot.pypy.org  Thu Oct 24 10:03:47 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 10:03:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: C types 'float' and 'long double'
Message-ID: <20131024080347.0864A1C0204@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67551:1532c2508a7c
Date: 2013-10-24 10:03 +0200
http://bitbucket.org/pypy/pypy/changeset/1532c2508a7c/

Log:	C types 'float' and 'long double'

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -369,13 +369,17 @@
         return None
 
     def pack_list_of_items(self, cdata, w_ob):
-        if self.size == rffi.sizeof(rffi.DOUBLE): # XXX
-            float_list = self.space.listview_float(w_ob)
-            if float_list is not None:
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
                 from rpython.rlib.rarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.DOUBLEP, cdata)
                 copy_list_to_raw_array(float_list, cdata)
                 return True
+            elif self.size == rffi.sizeof(rffi.FLOAT):
+                misc.pack_float_list_to_raw_array(float_list, cdata,
+                                                  rffi.FLOAT, rffi.FLOATP)
+                return True
         return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
 
 
@@ -431,3 +435,11 @@
         else:
             value = space.float_w(space.float(w_ob))
             self._to_longdouble_and_write(value, cdata)
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            misc.pack_float_list_to_raw_array(float_list, cdata,
+                                             rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -329,3 +329,10 @@
                 ptr[i] = rffi.cast(TP, x)
             return 0
     raise NotImplementedError("bad integer size")
+
+ at specialize.arg(2)
+def pack_float_list_to_raw_array(float_list, target, TP, TPP):
+    target = rffi.cast(TPP, target)
+    for i in range(len(float_list)):
+        x = float_list[i]
+        target[i] = rffi.cast(TP, x)
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -87,6 +87,24 @@
         raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-1])
         raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-sys.maxint])
 
+    def test_fast_init_cfloat_from_list(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, None)
+        buf = _cffi_backend.newp(FLOAT_ARRAY, [1.25, -3.5])
+        assert buf[0] == 1.25
+        assert buf[1] == -3.5
+
+    def test_fast_init_clongdouble_from_list(self):
+        import _cffi_backend
+        LONGDOUBLE = _cffi_backend.new_primitive_type('long double')
+        P_LONGDOUBLE = _cffi_backend.new_pointer_type(LONGDOUBLE)
+        LONGDOUBLE_ARRAY = _cffi_backend.new_array_type(P_LONGDOUBLE, None)
+        buf = _cffi_backend.newp(LONGDOUBLE_ARRAY, [1.25, -3.5])
+        assert float(buf[0]) == 1.25
+        assert float(buf[1]) == -3.5
+
 
 class AppTest_fast_path_bug(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))

From noreply at buildbot.pypy.org  Thu Oct 24 10:12:14 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 10:12:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: Unpacking lists of smaller integer types
Message-ID: <20131024081214.9495F1C00D8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67552:41853fcb0f1b
Date: 2013-10-24 10:11 +0200
http://bitbucket.org/pypy/pypy/changeset/41853fcb0f1b/

Log:	Unpacking lists of smaller integer types

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -203,13 +203,17 @@
         w_cdata.write_raw_signed_data(value)
 
     def unpack_list_of_int_items(self, w_cdata):
-        if self.size == rffi.sizeof(rffi.LONG): # XXX
+        if self.size == rffi.sizeof(rffi.LONG):
             from rpython.rlib.rarray import populate_list_from_raw_array
             res = []
             buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
             length = w_cdata.get_array_length()
             populate_list_from_raw_array(res, buf, length)
             return res
+        elif self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
+            return res
         return None
 
     def pack_list_of_items(self, cdata, w_ob):
@@ -284,6 +288,13 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_unsigned_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
+            return res
+        return None
+
     def pack_list_of_items(self, cdata, w_ob):
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -336,3 +336,12 @@
     for i in range(len(float_list)):
         x = float_list[i]
         target[i] = rffi.cast(TP, x)
+
+def unpack_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -138,12 +138,20 @@
         self._original = original
         rarray.populate_list_from_raw_array = populate_list_from_raw_array
         #
+        original2 = misc.unpack_list_from_raw_array
+        def unpack_list_from_raw_array(*args):
+            self.count += 1
+            return original2(*args)
+        self._original2 = original2
+        misc.unpack_list_from_raw_array = unpack_list_from_raw_array
+        #
         self.w_runappdirect = self.space.wrap(self.runappdirect)
 
 
     def teardown_method(self, meth):
         from rpython.rlib import rarray
         rarray.populate_list_from_raw_array = self._original
+        misc.unpack_list_from_raw_array = self._original2
 
     def test_list_int(self):
         import _cffi_backend
@@ -190,3 +198,31 @@
         assert lst == [1.1, 2.2, 3.3]
         if not self.runappdirect:
             assert self.get_count() == 1
+
+    def test_list_short(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, 3)
+        buf = _cffi_backend.newp(SHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_ushort(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, 3)
+        buf = _cffi_backend.newp(USHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        if not self.runappdirect:
+            assert self.get_count() == 1

From noreply at buildbot.pypy.org  Thu Oct 24 10:23:37 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 10:23:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: list(array-of-c-floats).
Message-ID: <20131024082337.5CC5E1C11BF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67553:b8da183e86ae
Date: 2013-10-24 10:17 +0200
http://bitbucket.org/pypy/pypy/changeset/b8da183e86ae/

Log:	list(array-of-c-floats).

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -370,13 +370,17 @@
         misc.write_raw_float_data(cdata, value, self.size)
 
     def unpack_list_of_float_items(self, w_cdata):
-        if self.size == rffi.sizeof(rffi.DOUBLE): # XXX
+        if self.size == rffi.sizeof(rffi.DOUBLE):
             from rpython.rlib.rarray import populate_list_from_raw_array
             res = []
             buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
             length = w_cdata.get_array_length()
             populate_list_from_raw_array(res, buf, length)
             return res
+        elif self.size == rffi.sizeof(rffi.FLOAT):
+            res = [0.0] * w_cdata.get_array_length()
+            misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata)
+            return res
         return None
 
     def pack_list_of_items(self, cdata, w_ob):
@@ -447,6 +451,10 @@
             value = space.float_w(space.float(w_ob))
             self._to_longdouble_and_write(value, cdata)
 
+    # Cannot have unpack_list_of_float_items() here:
+    # 'list(array-of-longdouble)' returns a list of cdata objects,
+    # not a list of floats.
+
     def pack_list_of_items(self, cdata, w_ob):
         float_list = self.space.listview_float(w_ob)
         if float_list is not None:
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -345,3 +345,8 @@
                 int_list[i] = rffi.cast(lltype.Signed, ptr[i])
             return
     raise NotImplementedError("bad integer size")
+
+def unpack_cfloat_list_from_raw_array(float_list, source):
+    ptr = rffi.cast(rffi.FLOATP, source)
+    for i in range(len(float_list)):
+        float_list[i] = rffi.cast(lltype.Float, ptr[i])
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -145,6 +145,14 @@
         self._original2 = original2
         misc.unpack_list_from_raw_array = unpack_list_from_raw_array
         #
+        original3 = misc.unpack_cfloat_list_from_raw_array
+        def unpack_cfloat_list_from_raw_array(*args):
+            self.count += 1
+            return original3(*args)
+        self._original3 = original3
+        misc.unpack_cfloat_list_from_raw_array = (
+            unpack_cfloat_list_from_raw_array)
+        #
         self.w_runappdirect = self.space.wrap(self.runappdirect)
 
 
@@ -152,6 +160,7 @@
         from rpython.rlib import rarray
         rarray.populate_list_from_raw_array = self._original
         misc.unpack_list_from_raw_array = self._original2
+        misc.unpack_cfloat_list_from_raw_array = self._original3
 
     def test_list_int(self):
         import _cffi_backend
@@ -226,3 +235,17 @@
         assert lst == [1, 2, 3]
         if not self.runappdirect:
             assert self.get_count() == 1
+
+    def test_list_cfloat(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, 3)
+        buf = _cffi_backend.newp(FLOAT_ARRAY)
+        buf[0] = 1.25
+        buf[1] = -2.5
+        buf[2] = 3.75
+        lst = list(buf)
+        assert lst == [1.25, -2.5, 3.75]
+        if not self.runappdirect:
+            assert self.get_count() == 1

From noreply at buildbot.pypy.org  Thu Oct 24 10:23:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 10:23:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: test and fix
Message-ID: <20131024082338.D9B231C11BF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67554:20c63568140f
Date: 2013-10-24 10:22 +0200
http://bitbucket.org/pypy/pypy/changeset/20c63568140f/

Log:	test and fix

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -291,7 +291,8 @@
     def unpack_list_of_int_items(self, w_cdata):
         if self.value_fits_long:
             res = [0] * w_cdata.get_array_length()
-            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
+            misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata,
+                                                     self.size)
             return res
         return None
 
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -346,6 +346,15 @@
             return
     raise NotImplementedError("bad integer size")
 
+def unpack_unsigned_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_unsigned_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
 def unpack_cfloat_list_from_raw_array(float_list, source):
     ptr = rffi.cast(rffi.FLOATP, source)
     for i in range(len(float_list)):
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -153,6 +153,14 @@
         misc.unpack_cfloat_list_from_raw_array = (
             unpack_cfloat_list_from_raw_array)
         #
+        original4 = misc.unpack_unsigned_list_from_raw_array
+        def unpack_unsigned_list_from_raw_array(*args):
+            self.count += 1
+            return original4(*args)
+        self._original4 = original4
+        misc.unpack_unsigned_list_from_raw_array = (
+            unpack_unsigned_list_from_raw_array)
+        #
         self.w_runappdirect = self.space.wrap(self.runappdirect)
 
 
@@ -161,6 +169,7 @@
         rarray.populate_list_from_raw_array = self._original
         misc.unpack_list_from_raw_array = self._original2
         misc.unpack_cfloat_list_from_raw_array = self._original3
+        misc.unpack_unsigned_list_from_raw_array = self._original4
 
     def test_list_int(self):
         import _cffi_backend
@@ -230,9 +239,9 @@
         buf = _cffi_backend.newp(USHORT_ARRAY)
         buf[0] = 1
         buf[1] = 2
-        buf[2] = 3
+        buf[2] = 50505
         lst = list(buf)
-        assert lst == [1, 2, 3]
+        assert lst == [1, 2, 50505]
         if not self.runappdirect:
             assert self.get_count() == 1
 

From noreply at buildbot.pypy.org  Thu Oct 24 11:23:42 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 11:23:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: an attempt to make it faster to compute
 inequality of some stuff
Message-ID: <20131024092342.12FFA1C11BF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67555:72c1e31475e9
Date: 2013-10-23 14:28 +0200
http://bitbucket.org/pypy/pypy/changeset/72c1e31475e9/

Log:	an attempt to make it faster to compute inequality of some stuff

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -97,8 +97,13 @@
     def __eq__(self, other):
         if isinstance(other, Typedef):
             return other.__eq__(self)
-        return self.__class__ is other.__class__ and (
-            self is other or safe_equal(self.__dict__, other.__dict__))
+        if self.__class__ is other.__class__:
+            if self is other:
+                return True
+            if hash(self) != hash(other):
+                return False
+            return safe_equal(self.__dict__, other.__dict__)
+        return False
 
     def __ne__(self, other):
         return not (self == other)
@@ -227,6 +232,9 @@
         self.OF = OF
         self.c_name = c_name
 
+    def __hash__(self):
+        return hash(self.OF)
+
     def __repr__(self):
         return '<Typedef "%s" of %r>' % (self.c_name, self.OF)
 

From noreply at buildbot.pypy.org  Thu Oct 24 11:23:43 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 11:23:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix
Message-ID: <20131024092343.6FB5A1C11BF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67556:d1ee8abf66b0
Date: 2013-10-23 14:40 +0200
http://bitbucket.org/pypy/pypy/changeset/d1ee8abf66b0/

Log:	fix

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -100,8 +100,11 @@
         if self.__class__ is other.__class__:
             if self is other:
                 return True
-            if hash(self) != hash(other):
-                return False
+            try:
+                if hash(self) != hash(other):
+                    return False
+            except TypeError:
+                pass # too bad, we can't use a fastpath here
             return safe_equal(self.__dict__, other.__dict__)
         return False
 

From noreply at buildbot.pypy.org  Thu Oct 24 11:23:44 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 11:23:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: A controversial checkin - try to limit
 the size of write analyzer analysis
Message-ID: <20131024092344.981051C11BF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67557:c89621c833be
Date: 2013-10-24 11:21 +0200
http://bitbucket.org/pypy/pypy/changeset/c89621c833be/

Log:	A controversial checkin - try to limit the size of write analyzer
	analysis

diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -4,6 +4,7 @@
 top_set = object()
 empty_set = frozenset()
 
+CUTOFF = 1000
 
 class WriteAnalyzer(graphanalyze.GraphAnalyzer):
     def bottom_result(self):
@@ -21,6 +22,8 @@
     def add_to_result(self, result, other):
         if other is top_set:
             return top_set
+        if len(other) + len(result) > CUTOFF:
+            return top_set
         result.update(other)
         return result
 

From noreply at buildbot.pypy.org  Thu Oct 24 11:23:45 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 11:23:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: try to reuse the 'seen' of
	readwriteanalyzer
Message-ID: <20131024092345.B1DCD1C11BF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67558:637a1abece5d
Date: 2013-10-24 11:21 +0200
http://bitbucket.org/pypy/pypy/changeset/637a1abece5d/

Log:	try to reuse the 'seen' of readwriteanalyzer

diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -11,6 +11,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
+from rpython.translator.backendopt.graphanalyze import DependencyTracker
 
 
 class CallControl(object):
@@ -35,6 +36,7 @@
         #
         for index, jd in enumerate(jitdrivers_sd):
             jd.index = index
+        self.seen = DependencyTracker(self.readwrite_analyzer)
 
     def find_all_graphs(self, policy):
         try:
@@ -231,8 +233,8 @@
                 extraeffect = EffectInfo.EF_CANNOT_RAISE
         #
         effectinfo = effectinfo_from_writeanalyze(
-            self.readwrite_analyzer.analyze(op), self.cpu, extraeffect,
-            oopspecindex, can_invalidate, call_release_gil_target,
+            self.readwrite_analyzer.analyze(op, self.seen), self.cpu,
+            extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
         )
         #
         assert effectinfo is not None

From noreply at buildbot.pypy.org  Thu Oct 24 11:23:47 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 11:23:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131024092347.32CFD1C11BF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67559:c480a86269d4
Date: 2013-10-24 11:23 +0200
http://bitbucket.org/pypy/pypy/changeset/c480a86269d4/

Log:	merge

diff --git a/lib_pypy/numpypy/lib/utils.py b/lib_pypy/numpypy/lib/utils.py
--- a/lib_pypy/numpypy/lib/utils.py
+++ b/lib_pypy/numpypy/lib/utils.py
@@ -21,14 +21,4 @@
         ...
 
     """
-    try:
-        import numpy
-    except:
-        # running from pypy source directory
-        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-        return os.path.join(head, '../include')
-    else:
-        # using installed numpy core headers
-        import numpy.core as core
-        d = os.path.join(os.path.dirname(core.__file__), 'include')
-    return d
+    return os.path.join(os.path.dirname(__file__), '../../../include')
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -283,10 +283,18 @@
         return self.ctype.iter(self)
 
     def unpackiterable_int(self, space):
-        return self.ctype.aslist_int(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_int_items(self)
+        return None
 
     def unpackiterable_float(self, space):
-        return self.ctype.aslist_float(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_float_items(self)
+        return None
 
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -105,26 +105,6 @@
     def iter(self, cdata):
         return W_CDataIter(self.space, self.ctitem, cdata)
 
-    def aslist_int(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_long():
-            res = []
-            buf = rffi.cast(rffi.LONGP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
-    def aslist_float(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_double():
-            res = []
-            buf = rffi.cast(rffi.DOUBLEP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
     def get_vararg_type(self):
         return self.ctptr
 
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -43,10 +43,13 @@
     def is_unichar_ptr_or_array(self):
         return False
 
-    def is_long(self):
-        return False
+    def unpack_list_of_int_items(self, cdata):
+        return None
 
-    def is_double(self):
+    def unpack_list_of_float_items(self, cdata):
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
         return False
 
     def newp(self, w_init):
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -2,6 +2,7 @@
 Primitives.
 """
 
+import sys
 from pypy.interpreter.error import operationerrfmt
 
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
@@ -170,9 +171,6 @@
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
 
-    def is_long(self):
-        return self.size == rffi.sizeof(lltype.Signed)
-
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
 
@@ -204,6 +202,41 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_signed_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.LONG):
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            if self.size == rffi.sizeof(rffi.LONG): # fastest path
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.LONGP, cdata)
+                copy_list_to_raw_array(int_list, cdata)
+            else:
+                if self.value_fits_long:
+                    vmin = self.vmin
+                    vrangemax = self.vrangemax
+                else:
+                    vmin = r_uint(0)
+                    vrangemax = r_uint(-1)
+                overflowed = misc.pack_list_to_raw_array_bounds(
+                    int_list, cdata, self.size, vmin, vrangemax)
+                if overflowed != 0:
+                    self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
     _attrs_            = ['value_fits_long', 'value_fits_ulong', 'vrangemax']
@@ -255,6 +288,28 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_unsigned_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata,
+                                                     self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            if self.value_fits_long:
+                vrangemax = self.vrangemax
+            else:
+                vrangemax = r_uint(sys.maxint)
+            overflowed = misc.pack_list_to_raw_array_bounds(
+                int_list, cdata, self.size, r_uint(0), vrangemax)
+            if overflowed != 0:
+                self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
     _attrs_ = []
@@ -276,9 +331,6 @@
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
 
-    def is_double(self):
-        return self.size == rffi.sizeof(lltype.Float)
-
     def cast(self, w_ob):
         space = self.space
         if isinstance(w_ob, cdataobj.W_CData):
@@ -318,6 +370,34 @@
         value = space.float_w(space.float(w_ob))
         misc.write_raw_float_data(cdata, value, self.size)
 
+    def unpack_list_of_float_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.DOUBLE):
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.size == rffi.sizeof(rffi.FLOAT):
+            res = [0.0] * w_cdata.get_array_length()
+            misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.DOUBLEP, cdata)
+                copy_list_to_raw_array(float_list, cdata)
+                return True
+            elif self.size == rffi.sizeof(rffi.FLOAT):
+                misc.pack_float_list_to_raw_array(float_list, cdata,
+                                                  rffi.FLOAT, rffi.FLOATP)
+                return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveLongDouble(W_CTypePrimitiveFloat):
     _attrs_ = []
@@ -328,7 +408,6 @@
         return misc.longdouble2str(lvalue)
 
     def cast(self, w_ob):
-        space = self.space
         if (isinstance(w_ob, cdataobj.W_CData) and
                 isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)):
             w_cdata = self.convert_to_object(w_ob._cdata)
@@ -372,3 +451,15 @@
         else:
             value = space.float_w(space.float(w_ob))
             self._to_longdouble_and_write(value, cdata)
+
+    # Cannot have unpack_list_of_float_items() here:
+    # 'list(array-of-longdouble)' returns a list of cdata objects,
+    # not a list of floats.
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            misc.pack_float_list_to_raw_array(float_list, cdata,
+                                             rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -42,12 +42,6 @@
     def is_char_or_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveCharOrUniChar)
 
-    def aslist_int(self, cdata):
-        return None
-
-    def aslist_float(self, cdata):
-        return None
-
     def cast(self, w_ob):
         # cast to a pointer, to a funcptr, or to an array.
         # Note that casting to an array is an extension to the C language,
@@ -64,24 +58,10 @@
             value = rffi.cast(rffi.CCHARP, value)
         return cdataobj.W_CData(space, value, self)
 
-    def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
-        from rpython.rlib.rarray import copy_list_to_raw_array
-        int_list = self.space.listview_int(w_ob)
-        float_list = self.space.listview_float(w_ob)
+    def _convert_array_from_listview(self, cdata, w_ob):
+        if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
+            return
         #
-        if self.ctitem.is_long() and int_list is not None:
-            cdata = rffi.cast(rffi.LONGP, cdata)
-            copy_list_to_raw_array(int_list, cdata)
-            return True
-        #
-        if self.ctitem.is_double() and float_list is not None:
-            cdata = rffi.cast(rffi.DOUBLEP, cdata)
-            copy_list_to_raw_array(float_list, cdata)
-            return True
-        #
-        return False
-
-    def _convert_array_from_listview(self, cdata, w_ob):
         space = self.space
         lst_w = space.listview(w_ob)
         if self.length >= 0 and len(lst_w) > self.length:
@@ -95,11 +75,6 @@
 
     def convert_array_from_object(self, cdata, w_ob):
         space = self.space
-        if self._convert_array_from_list_strategy_maybe(cdata, w_ob):
-            # the fast path worked, we are done now
-            return
-        #
-        # continue with the slow path
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             self._convert_array_from_listview(cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -315,3 +315,47 @@
             _raw_memclear_tp(TP, TPP, dest)
             return
     raise NotImplementedError("bad clear size")
+
+# ____________________________________________________________
+
+def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) - vmin > vrangemax:
+                    return x      # overflow
+                ptr[i] = rffi.cast(TP, x)
+            return 0
+    raise NotImplementedError("bad integer size")
+
+ at specialize.arg(2)
+def pack_float_list_to_raw_array(float_list, target, TP, TPP):
+    target = rffi.cast(TPP, target)
+    for i in range(len(float_list)):
+        x = float_list[i]
+        target[i] = rffi.cast(TP, x)
+
+def unpack_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_unsigned_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_unsigned_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_cfloat_list_from_raw_array(float_list, source):
+    ptr = rffi.cast(rffi.FLOATP, source)
+    for i in range(len(float_list)):
+        float_list[i] = rffi.cast(lltype.Float, ptr[i])
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -117,13 +117,17 @@
 
 SF_MSVC_BITFIELDS = 1
 SF_GCC_ARM_BITFIELDS = 2
+SF_GCC_BIG_ENDIAN = 4
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
-elif rffi_platform.getdefined('__arm__', ''):
-    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
-    DEFAULT_SFLAGS = 0
+    if rffi_platform.getdefined('__arm__', ''):
+        DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
+    else:
+        DEFAULT_SFLAGS = 0
+    if sys.byteorder == 'big':
+        DEFAULT_SFLAGS |= SF_GCC_BIG_ENDIAN
 
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
@@ -325,6 +329,9 @@
                     prev_bitfield_free -= fbitsize
                     field_offset_bytes = boffset / 8 - ftype.size
 
+                if sflags & SF_GCC_BIG_ENDIAN:
+                    bitshift = 8 * ftype.size - fbitsize- bitshift
+
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
                                            bitshift, fbitsize)
                 fields_list.append(fld)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2949,8 +2949,6 @@
     _test_bitfield_details(flag=2)
 
 def test_bitfield_as_big_endian():
-    if '__pypy__' in sys.builtin_module_names:
-        py.test.skip("no big endian machine supported on pypy for now")
     _test_bitfield_details(flag=4)
 
 
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -1,18 +1,19 @@
-# side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule()
+# side-effect: FORMAT_LONGDOUBLE must be built before the first test
 from pypy.module._cffi_backend import misc
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeobj import W_CType
+
 
 class AppTest_fast_path_from_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
 
     def setup_method(self, meth):
-        def forbidden(self, *args):
+        def forbidden(*args):
             assert False, 'The slow path is forbidden'
-        self._original = W_CTypePtrOrArray._convert_array_from_listview.im_func
-        W_CTypePtrOrArray._convert_array_from_listview = forbidden
+        self._original = W_CType.pack_list_of_items.im_func
+        W_CType.pack_list_of_items = forbidden
 
     def teardown_method(self, meth):
-        W_CTypePtrOrArray._convert_array_from_listview = self._original
+        W_CType.pack_list_of_items = self._original
 
     def test_fast_init_from_list(self):
         import _cffi_backend
@@ -34,6 +35,89 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
+    def test_fast_init_short_from_list(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, None)
+        buf = _cffi_backend.newp(SHORT_ARRAY, [1, -2, 3])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [40000])
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [-40000])
+
+    def test_fast_init_longlong_from_list(self):
+        if type(2 ** 50) is long:
+            large_int = 2 ** 30
+        else:
+            large_int = 2 ** 50
+        import _cffi_backend
+        LONGLONG = _cffi_backend.new_primitive_type('long long')
+        P_LONGLONG = _cffi_backend.new_pointer_type(LONGLONG)
+        LONGLONG_ARRAY = _cffi_backend.new_array_type(P_LONGLONG, None)
+        buf = _cffi_backend.newp(LONGLONG_ARRAY, [1, -2, 3, large_int])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        assert buf[3] == large_int
+
+    def test_fast_init_ushort_from_list(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, None)
+        buf = _cffi_backend.newp(USHORT_ARRAY, [1, 2, 40000])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == 40000
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [70000])
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [-1])
+
+    def test_fast_init_ulong_from_list(self):
+        import sys
+        import _cffi_backend
+        ULONG = _cffi_backend.new_primitive_type('unsigned long')
+        P_ULONG = _cffi_backend.new_pointer_type(ULONG)
+        ULONG_ARRAY = _cffi_backend.new_array_type(P_ULONG, None)
+        buf = _cffi_backend.newp(ULONG_ARRAY, [1, 2, sys.maxint])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == sys.maxint
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-1])
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-sys.maxint])
+
+    def test_fast_init_cfloat_from_list(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, None)
+        buf = _cffi_backend.newp(FLOAT_ARRAY, [1.25, -3.5])
+        assert buf[0] == 1.25
+        assert buf[1] == -3.5
+
+    def test_fast_init_clongdouble_from_list(self):
+        import _cffi_backend
+        LONGDOUBLE = _cffi_backend.new_primitive_type('long double')
+        P_LONGDOUBLE = _cffi_backend.new_pointer_type(LONGDOUBLE)
+        LONGDOUBLE_ARRAY = _cffi_backend.new_array_type(P_LONGDOUBLE, None)
+        buf = _cffi_backend.newp(LONGDOUBLE_ARRAY, [1.25, -3.5])
+        assert float(buf[0]) == 1.25
+        assert float(buf[1]) == -3.5
+
+
+class AppTest_fast_path_bug(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def test_bug_not_list_or_tuple(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY_2 = _cffi_backend.new_array_type(P_LONG, 2)
+        P_LONG_ARRAY_2 = _cffi_backend.new_pointer_type(LONG_ARRAY_2)
+        LONG_ARRAY_ARRAY = _cffi_backend.new_array_type(P_LONG_ARRAY_2, None)
+        raises(TypeError, _cffi_backend.newp, LONG_ARRAY_ARRAY, [set([4, 5])])
+
 
 class AppTest_fast_path_to_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
@@ -54,12 +138,38 @@
         self._original = original
         rarray.populate_list_from_raw_array = populate_list_from_raw_array
         #
+        original2 = misc.unpack_list_from_raw_array
+        def unpack_list_from_raw_array(*args):
+            self.count += 1
+            return original2(*args)
+        self._original2 = original2
+        misc.unpack_list_from_raw_array = unpack_list_from_raw_array
+        #
+        original3 = misc.unpack_cfloat_list_from_raw_array
+        def unpack_cfloat_list_from_raw_array(*args):
+            self.count += 1
+            return original3(*args)
+        self._original3 = original3
+        misc.unpack_cfloat_list_from_raw_array = (
+            unpack_cfloat_list_from_raw_array)
+        #
+        original4 = misc.unpack_unsigned_list_from_raw_array
+        def unpack_unsigned_list_from_raw_array(*args):
+            self.count += 1
+            return original4(*args)
+        self._original4 = original4
+        misc.unpack_unsigned_list_from_raw_array = (
+            unpack_unsigned_list_from_raw_array)
+        #
         self.w_runappdirect = self.space.wrap(self.runappdirect)
 
 
     def teardown_method(self, meth):
         from rpython.rlib import rarray
         rarray.populate_list_from_raw_array = self._original
+        misc.unpack_list_from_raw_array = self._original2
+        misc.unpack_cfloat_list_from_raw_array = self._original3
+        misc.unpack_unsigned_list_from_raw_array = self._original4
 
     def test_list_int(self):
         import _cffi_backend
@@ -84,6 +194,14 @@
         pbuf = _cffi_backend.cast(P_LONG, buf)
         raises(TypeError, "list(pbuf)")
 
+    def test_bug(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        five = _cffi_backend.cast(LONG, 5)
+        raises(TypeError, list, five)
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
+        raises(TypeError, list, five_and_a_half)
 
     def test_list_float(self):
         import _cffi_backend
@@ -98,3 +216,45 @@
         assert lst == [1.1, 2.2, 3.3]
         if not self.runappdirect:
             assert self.get_count() == 1
+
+    def test_list_short(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, 3)
+        buf = _cffi_backend.newp(SHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_ushort(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, 3)
+        buf = _cffi_backend.newp(USHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 50505
+        lst = list(buf)
+        assert lst == [1, 2, 50505]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_cfloat(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, 3)
+        buf = _cffi_backend.newp(FLOAT_ARRAY)
+        buf[0] = 1.25
+        buf[1] = -2.5
+        buf[2] = 3.75
+        lst = list(buf)
+        assert lst == [1.25, -2.5, 3.75]
+        if not self.runappdirect:
+            assert self.get_count() == 1
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -84,6 +84,7 @@
 
     def build_and_convert(self, space, box):
         return self.itemtype.build_and_convert(space, self, box)
+
     def coerce(self, space, w_item):
         return self.itemtype.coerce(space, self, w_item)
 
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -140,7 +140,7 @@
         ltime = rctime.localtime()
         assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
         try:
-            assert rctime.asctime((12345,) + (0,) * 8).split()[-1] == '12345'
+            rctime.asctime((12345,) + (0,) * 8)  # assert this doesn't crash
         except ValueError:
             pass  # some OS (ie POSIXes besides Linux) reject year > 9999
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1398,7 +1398,7 @@
                     assembler_call_jd)
             if resbox is not None:
                 self.make_result_of_lastop(resbox)
-            self.metainterp.vable_after_residual_call()
+            self.metainterp.vable_after_residual_call(funcbox)
             self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None)
             if vablebox is not None:
                 self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None)
@@ -2437,7 +2437,7 @@
                 # it by ConstPtr(NULL).
                 self.stop_tracking_virtualref(i)
 
-    def vable_after_residual_call(self):
+    def vable_after_residual_call(self, funcbox):
         vinfo = self.jitdriver_sd.virtualizable_info
         if vinfo is not None:
             virtualizable_box = self.virtualizable_boxes[-1]
@@ -2445,6 +2445,14 @@
             if vinfo.tracing_after_residual_call(virtualizable):
                 # the virtualizable escaped during CALL_MAY_FORCE.
                 self.load_fields_from_virtualizable()
+                target_name = self.staticdata.get_name_from_address(funcbox.getaddr())
+                if target_name:
+                    target_name = "ConstClass(%s)" % target_name
+                else:
+                    target_name = str(funcbox.getaddr())
+                debug_print('vable escaped during a call in %s to %s' % (
+                    self.framestack[-1].jitcode.name, target_name
+                ))
                 raise SwitchToBlackhole(Counters.ABORT_ESCAPE,
                                         raising_exception=True)
                 # ^^^ we set 'raising_exception' to True because we must still

From noreply at buildbot.pypy.org  Thu Oct 24 11:38:24 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 11:38:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: getfield_gc_pure is just like getfield
 - does not escape it's args
Message-ID: <20131024093824.D53631C0204@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67560:71f7e47d44c1
Date: 2013-10-24 11:37 +0200
http://bitbucket.org/pypy/pypy/changeset/71f7e47d44c1/

Log:	getfield_gc_pure is just like getfield - does not escape it's args

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -82,6 +82,7 @@
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
         elif (opnum != rop.GETFIELD_GC and
+              opnum != rop.GETFIELD_GC_PURE and
               opnum != rop.MARK_OPAQUE_PTR and
               opnum != rop.PTR_EQ and
               opnum != rop.PTR_NE and

From noreply at buildbot.pypy.org  Thu Oct 24 11:45:24 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 11:45:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a test for arrays of bools.
Message-ID: <20131024094524.63F201C135D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67561:a0518eb1507f
Date: 2013-10-24 11:44 +0200
http://bitbucket.org/pypy/pypy/changeset/a0518eb1507f/

Log:	Add a test for arrays of bools.

diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -105,6 +105,18 @@
         assert float(buf[0]) == 1.25
         assert float(buf[1]) == -3.5
 
+    def test_fast_init_bool_from_list(self):
+        import _cffi_backend
+        BOOL = _cffi_backend.new_primitive_type('_Bool')
+        P_BOOL = _cffi_backend.new_pointer_type(BOOL)
+        BOOL_ARRAY = _cffi_backend.new_array_type(P_BOOL, None)
+        buf = _cffi_backend.newp(BOOL_ARRAY, [1, 0])
+        assert buf[0] == 1
+        assert buf[1] == 0
+        assert type(buf[1]) is int
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [2])
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [-1])
+
 
 class AppTest_fast_path_bug(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))

From noreply at buildbot.pypy.org  Thu Oct 24 11:45:25 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 11:45:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20131024094525.C74291C135D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67562:8d2f699e2f5b
Date: 2013-10-24 11:44 +0200
http://bitbucket.org/pypy/pypy/changeset/8d2f699e2f5b/

Log:	merge heads

diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -11,6 +11,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
+from rpython.translator.backendopt.graphanalyze import DependencyTracker
 
 
 class CallControl(object):
@@ -35,6 +36,7 @@
         #
         for index, jd in enumerate(jitdrivers_sd):
             jd.index = index
+        self.seen = DependencyTracker(self.readwrite_analyzer)
 
     def find_all_graphs(self, policy):
         try:
@@ -231,8 +233,8 @@
                 extraeffect = EffectInfo.EF_CANNOT_RAISE
         #
         effectinfo = effectinfo_from_writeanalyze(
-            self.readwrite_analyzer.analyze(op), self.cpu, extraeffect,
-            oopspecindex, can_invalidate, call_release_gil_target,
+            self.readwrite_analyzer.analyze(op, self.seen), self.cpu,
+            extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
         )
         #
         assert effectinfo is not None
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -82,6 +82,7 @@
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
         elif (opnum != rop.GETFIELD_GC and
+              opnum != rop.GETFIELD_GC_PURE and
               opnum != rop.MARK_OPAQUE_PTR and
               opnum != rop.PTR_EQ and
               opnum != rop.PTR_NE and
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -97,8 +97,16 @@
     def __eq__(self, other):
         if isinstance(other, Typedef):
             return other.__eq__(self)
-        return self.__class__ is other.__class__ and (
-            self is other or safe_equal(self.__dict__, other.__dict__))
+        if self.__class__ is other.__class__:
+            if self is other:
+                return True
+            try:
+                if hash(self) != hash(other):
+                    return False
+            except TypeError:
+                pass # too bad, we can't use a fastpath here
+            return safe_equal(self.__dict__, other.__dict__)
+        return False
 
     def __ne__(self, other):
         return not (self == other)
@@ -227,6 +235,9 @@
         self.OF = OF
         self.c_name = c_name
 
+    def __hash__(self):
+        return hash(self.OF)
+
     def __repr__(self):
         return '<Typedef "%s" of %r>' % (self.c_name, self.OF)
 
diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -4,6 +4,7 @@
 top_set = object()
 empty_set = frozenset()
 
+CUTOFF = 1000
 
 class WriteAnalyzer(graphanalyze.GraphAnalyzer):
     def bottom_result(self):
@@ -21,6 +22,8 @@
     def add_to_result(self, result, other):
         if other is top_set:
             return top_set
+        if len(other) + len(result) > CUTOFF:
+            return top_set
         result.update(other)
         return result
 

From noreply at buildbot.pypy.org  Thu Oct 24 11:53:24 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 11:53:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: improve the test to check got
	GETFIELD_GC_PURE
Message-ID: <20131024095324.AD4991C356B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67563:bc1c47a6b9f2
Date: 2013-10-24 11:49 +0200
http://bitbucket.org/pypy/pypy/changeset/bc1c47a6b9f2/

Log:	improve the test to check got GETFIELD_GC_PURE

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3360,21 +3360,28 @@
         self.check_resops(call=0, getfield_gc=0)
 
     def test_isvirtual_call_assembler(self):
-        driver = JitDriver(greens = ['code'], reds = ['n'])
+        driver = JitDriver(greens = ['code'], reds = ['n', 's'])
 
         @look_inside_iff(lambda t1, t2: isvirtual(t1))
         def g(t1, t2):
             return t1[0] == t2[0]
 
+        def create(n):
+            return (1, 2, n)
+        create._dont_inline_ = True
+
         def f(code, n):
+            s = 0
             while n > 0:
-                driver.can_enter_jit(code=code, n=n)
-                driver.jit_merge_point(code=code, n=n)
-                t = (1, 2, n)
+                driver.can_enter_jit(code=code, n=n, s=s)
+                driver.jit_merge_point(code=code, n=n, s=s)
+                t = create(n)
                 if code:
                     f(0, 3)
+                s += t[2]
                 g(t, (1, 2, n))
                 n -= 1
+            return s
 
         self.meta_interp(f, [1, 10], inline=True)
         self.check_resops(call=0, call_may_force=0, call_assembler=2)

From noreply at buildbot.pypy.org  Thu Oct 24 11:53:25 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 11:53:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131024095325.EA9C41C357F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67564:214bfa391b33
Date: 2013-10-24 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/214bfa391b33/

Log:	merge

diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -105,6 +105,18 @@
         assert float(buf[0]) == 1.25
         assert float(buf[1]) == -3.5
 
+    def test_fast_init_bool_from_list(self):
+        import _cffi_backend
+        BOOL = _cffi_backend.new_primitive_type('_Bool')
+        P_BOOL = _cffi_backend.new_pointer_type(BOOL)
+        BOOL_ARRAY = _cffi_backend.new_array_type(P_BOOL, None)
+        buf = _cffi_backend.newp(BOOL_ARRAY, [1, 0])
+        assert buf[0] == 1
+        assert buf[1] == 0
+        assert type(buf[1]) is int
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [2])
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [-1])
+
 
 class AppTest_fast_path_bug(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))

From noreply at buildbot.pypy.org  Thu Oct 24 13:00:05 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 13:00:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: No-op,
 make sure that all instances of W_CTypePrimitiveXxx have all
Message-ID: <20131024110005.29E421C135D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67565:ae07e942ebdf
Date: 2013-10-24 12:59 +0200
http://bitbucket.org/pypy/pypy/changeset/ae07e942ebdf/

Log:	No-op, make sure that all instances of W_CTypePrimitiveXxx have all
	attributes.

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -170,6 +170,9 @@
             sh = self.size * 8
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
+        else:
+            self.vmin = r_uint(0)
+            self.vrangemax = r_uint(-1)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -224,14 +227,8 @@
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
-                if self.value_fits_long:
-                    vmin = self.vmin
-                    vrangemax = self.vrangemax
-                else:
-                    vmin = r_uint(0)
-                    vrangemax = r_uint(-1)
                 overflowed = misc.pack_list_to_raw_array_bounds(
-                    int_list, cdata, self.size, vmin, vrangemax)
+                    int_list, cdata, self.size, self.vmin, self.vrangemax)
                 if overflowed != 0:
                     self._overflow(self.space.wrap(overflowed))
             return True
@@ -249,6 +246,8 @@
         self.value_fits_ulong = self.size <= rffi.sizeof(lltype.Unsigned)
         if self.value_fits_long:
             self.vrangemax = self._compute_vrange_max()
+        else:
+            self.vrangemax = r_uint(sys.maxint)
 
     def _compute_vrange_max(self):
         sh = self.size * 8
@@ -299,12 +298,8 @@
     def pack_list_of_items(self, cdata, w_ob):
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
-            if self.value_fits_long:
-                vrangemax = self.vrangemax
-            else:
-                vrangemax = r_uint(sys.maxint)
             overflowed = misc.pack_list_to_raw_array_bounds(
-                int_list, cdata, self.size, r_uint(0), vrangemax)
+                int_list, cdata, self.size, r_uint(0), self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.wrap(overflowed))
             return True

From noreply at buildbot.pypy.org  Thu Oct 24 14:06:02 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 14:06:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: attack the first part of slowness of
 stackcheckinsertion. I still dont
Message-ID: <20131024120602.7A6E91C135D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67566:1a8ee9ab87d4
Date: 2013-10-24 14:04 +0200
http://bitbucket.org/pypy/pypy/changeset/1a8ee9ab87d4/

Log:	attack the first part of slowness of stackcheckinsertion. I still
	dont understand why we have >1mln edges though

diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py
--- a/rpython/rlib/rstack.py
+++ b/rpython/rlib/rstack.py
@@ -67,6 +67,7 @@
     # Else call the slow path
     stack_check_slowpath(current)
 stack_check._always_inline_ = True
+stack_check._dont_insert_stackcheck_ = True
 
 @rgc.no_collect
 def stack_check_slowpath(current):
@@ -74,3 +75,4 @@
         from rpython.rlib.rstackovf import _StackOverflow
         raise _StackOverflow
 stack_check_slowpath._dont_inline_ = True
+stack_check_slowpath._dont_insert_stackcheck_ = True
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -213,6 +213,10 @@
     insert_in = set()
     block2graph = {}
     for caller in translator.graphs:
+        pyobj = getattr(caller, 'func', None)
+        if pyobj is not None:
+            if getattr(pyobj, '_dont_insert_stackcheck_', False):
+                continue
         for block, callee in find_calls_from(translator, caller):
             if getattr(getattr(callee, 'func', None),
                        'insert_stack_check_here', False):
@@ -269,4 +273,4 @@
     transform_dead_op_vars(ann, block_subset)
     if ann.translator:
         checkgraphs(ann, block_subset)
- 
+

From noreply at buildbot.pypy.org  Thu Oct 24 14:06:03 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 14:06:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131024120603.D28301C1380@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67567:58a5cf1c9c5f
Date: 2013-10-24 14:05 +0200
http://bitbucket.org/pypy/pypy/changeset/58a5cf1c9c5f/

Log:	merge

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -170,6 +170,9 @@
             sh = self.size * 8
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
+        else:
+            self.vmin = r_uint(0)
+            self.vrangemax = r_uint(-1)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -224,14 +227,8 @@
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
-                if self.value_fits_long:
-                    vmin = self.vmin
-                    vrangemax = self.vrangemax
-                else:
-                    vmin = r_uint(0)
-                    vrangemax = r_uint(-1)
                 overflowed = misc.pack_list_to_raw_array_bounds(
-                    int_list, cdata, self.size, vmin, vrangemax)
+                    int_list, cdata, self.size, self.vmin, self.vrangemax)
                 if overflowed != 0:
                     self._overflow(self.space.wrap(overflowed))
             return True
@@ -249,6 +246,8 @@
         self.value_fits_ulong = self.size <= rffi.sizeof(lltype.Unsigned)
         if self.value_fits_long:
             self.vrangemax = self._compute_vrange_max()
+        else:
+            self.vrangemax = r_uint(sys.maxint)
 
     def _compute_vrange_max(self):
         sh = self.size * 8
@@ -299,12 +298,8 @@
     def pack_list_of_items(self, cdata, w_ob):
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
-            if self.value_fits_long:
-                vrangemax = self.vrangemax
-            else:
-                vrangemax = r_uint(sys.maxint)
             overflowed = misc.pack_list_to_raw_array_bounds(
-                int_list, cdata, self.size, r_uint(0), vrangemax)
+                int_list, cdata, self.size, r_uint(0), self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.wrap(overflowed))
             return True

From noreply at buildbot.pypy.org  Thu Oct 24 14:17:42 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 14:17:42 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: fight a bit with reprs and
	recursivness
Message-ID: <20131024121742.EF1441C135D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67568:b4aa204d4648
Date: 2013-10-24 14:16 +0200
http://bitbucket.org/pypy/pypy/changeset/b4aa204d4648/

Log:	fight a bit with reprs and recursivness

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -43,7 +43,8 @@
 def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
                 ll_fasthash_function=None, ll_hash_function=None,
                 ll_eq_function=None, method_cache={},
-                dummykeyobj=None, dummyvalueobj=None, rtyper=None):
+                dummykeyobj=None, dummyvalueobj=None, rtyper=None,
+                setup_lookup_funcs=True):
     # get the actual DICT type. if DICT is None, it's created, otherwise
     # forward reference is becoming DICT
     if DICT is None:
@@ -102,10 +103,6 @@
     LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY,
                                               lltype.Signed, lltype.Signed],
                                              lltype.Signed))
-    STORECLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
-                                                  lltype.Signed,
-                                                  lltype.Signed],
-                                                 lltype.Void))
 
     fields =          [ ("num_items", lltype.Signed),
                         ("num_used_items", lltype.Signed),
@@ -147,6 +144,16 @@
                                 *fields))
 
     family.empty_array = DICTENTRYARRAY.allocate(0)
+    if setup_lookup_funcs:
+        _setup_lookup_funcs(LOOKUP_FUNC, DICT, rtyper, family)
+    return DICT
+
+def _setup_lookup_funcs(LOOKUP_FUNC, DICT, rtyper, family):
+    STORECLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
+                                                  lltype.Signed,
+                                                  lltype.Signed],
+                                                 lltype.Void))
+
     for name, T in [('byte', rffi.UCHAR),
                     ('short', rffi.USHORT),
                     ('int', rffi.UINT),
@@ -158,7 +165,6 @@
                                                       rtyper=rtyper)
         setattr(family, '%s_lookup_function' % name, lookupfn)
         setattr(family, '%s_insert_clean_function' % name, storecleanfn)
-    return DICT
 
 def llhelper_or_compile(rtyper, FUNCPTR, ll_func):
     # the check is for pseudo rtyper from tests
@@ -226,9 +232,17 @@
             kwd['dummyvalueobj'] = self.value_repr.get_ll_dummyval_obj(
                 self.rtyper, s_value)
 
+            kwd['setup_lookup_funcs'] = False
             get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT,
                         rtyper=self.rtyper, **kwd)
 
+    def _setup_repr_final(self):
+        LOOKUP_FUNC = self.lowleveltype.TO.lookup_function
+        family = self.lowleveltype.TO.lookup_family
+        _setup_lookup_funcs(LOOKUP_FUNC, self.lowleveltype.TO, self.rtyper,
+                            family)
+
+
     def convert_const(self, dictobj):
         from rpython.rtyper.lltypesystem import llmemory
         # get object from bound dict methods

From noreply at buildbot.pypy.org  Thu Oct 24 14:17:44 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 14:17:44 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: merge
Message-ID: <20131024121744.332591C135D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67569:cb19a98c78ca
Date: 2013-10-24 14:17 +0200
http://bitbucket.org/pypy/pypy/changeset/cb19a98c78ca/

Log:	merge

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -1099,11 +1099,11 @@
 
     entries = dic.entries
 
-    i = dic.num_used_items - 1
     while True:
+        i = dic.num_used_items - 1
         if entries.valid(i):
             break
-        i -= 1
+        dic.num_used_items -= 1
 
     key = entries[i].key
     index = dic.lookup_function(dic, key, entries.hash(i),
@@ -1122,7 +1122,7 @@
     r = lltype.malloc(ELEM.TO)
     r.item0 = recast(ELEM.TO.item0, entry.key)
     r.item1 = recast(ELEM.TO.item1, entry.value)
-    _ll_dict_del(dic, r_uint(i))
+    _ll_dict_del(dic, i)
     return r
 
 def ll_dict_pop(dic, key):

From noreply at buildbot.pypy.org  Thu Oct 24 15:37:20 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 15:37:20 +0200 (CEST)
Subject: [pypy-commit] pypy rordereddict: a branch to import
 rdict-experiments-3 as rordereddict (and not touch
Message-ID: <20131024133720.E2AF11D23C8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rordereddict
Changeset: r67570:5ac11dc114fb
Date: 2013-10-24 14:46 +0200
http://bitbucket.org/pypy/pypy/changeset/5ac11dc114fb/

Log:	a branch to import rdict-experiments-3 as rordereddict (and not
	touch the dict implementation itself at all), thus making
	OrderedDict RPython


From noreply at buildbot.pypy.org  Thu Oct 24 15:37:22 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 15:37:22 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: Don't make dictionaries
 that have empty tuples as keys
Message-ID: <20131024133722.853011D23C8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67571:b6b80c7b692d
Date: 2013-10-24 15:35 +0200
http://bitbucket.org/pypy/pypy/changeset/b6b80c7b692d/

Log:	Don't make dictionaries that have empty tuples as keys

diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -123,7 +123,6 @@
     else:
         return rffi.cast(lltype.Signed, x)
 
-
 class JitCell(BaseJitCell):
     # the counter can mean the following things:
     #     counter >=  0: not yet traced, wait till threshold is reached
@@ -291,7 +290,7 @@
             if vinfo is not None:
                 virtualizable = args[index_of_virtualizable]
                 vinfo.clear_vable_token(virtualizable)
-            
+
             deadframe = func_execute_token(loop_token, *args)
             #
             # Record in the memmgr that we just ran this loop,
@@ -429,6 +428,12 @@
     def _make_jitcell_getter_default(self):
         "NOT_RPYTHON"
         jitdriver_sd = self.jitdriver_sd
+        if len(jitdriver_sd._green_args_spec) == 0:
+            prebuilt_jitcell = JitCell()
+
+            def get_jitcell(build):
+                return prebuilt_jitcell
+            return get_jitcell
         green_args_spec = unrolling_iterable(jitdriver_sd._green_args_spec)
         #
         def comparekey(greenargs1, greenargs2):

From noreply at buildbot.pypy.org  Thu Oct 24 15:37:23 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 15:37:23 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: Shuffle setup around until
	it works
Message-ID: <20131024133723.C1F391D23C8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67572:892b8d15b53f
Date: 2013-10-24 15:35 +0200
http://bitbucket.org/pypy/pypy/changeset/892b8d15b53f/

Log:	Shuffle setup around until it works

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -183,6 +183,7 @@
     def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
                  custom_eq_hash=None):
         self.rtyper = rtyper
+        self.finalized = False
         self.DICT = lltype.GcForwardReference()
         self.lowleveltype = lltype.Ptr(self.DICT)
         self.custom_eq_hash = custom_eq_hash is not None
@@ -237,10 +238,12 @@
                         rtyper=self.rtyper, **kwd)
 
     def _setup_repr_final(self):
-        LOOKUP_FUNC = self.lowleveltype.TO.lookup_function
-        family = self.lowleveltype.TO.lookup_family
-        _setup_lookup_funcs(LOOKUP_FUNC, self.lowleveltype.TO, self.rtyper,
-                            family)
+        if not self.finalized:
+            LOOKUP_FUNC = self.lowleveltype.TO.lookup_function
+            family = self.lowleveltype.TO.lookup_family
+            _setup_lookup_funcs(LOOKUP_FUNC, self.lowleveltype.TO, self.rtyper,
+                                family)
+            self.finalized = True
 
 
     def convert_const(self, dictobj):
@@ -256,6 +259,7 @@
             return self.dict_cache[key]
         except KeyError:
             self.setup()
+            self.setup_final()
             l_dict = ll_newdict_size(self.DICT, len(dictobj))
             self.dict_cache[key] = l_dict
             r_key = self.key_repr
diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py
--- a/rpython/rtyper/rmodel.py
+++ b/rpython/rtyper/rmodel.py
@@ -53,7 +53,7 @@
         self._initialized = setupstate.INPROGRESS
         try:
             self._setup_repr()
-        except TyperError, e:
+        except TyperError:
             self._initialized = setupstate.BROKEN
             raise
         else:

From noreply at buildbot.pypy.org  Thu Oct 24 17:10:34 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 17:10:34 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: implement fast function
 lookup (by hand a bit)
Message-ID: <20131024151034.AA3841C00D8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67573:3b14a46a82d9
Date: 2013-10-24 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/3b14a46a82d9/

Log:	implement fast function lookup (by hand a bit)

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -32,7 +32,7 @@
 #        int resize_counter;
 #        {byte, short, int, long} *indexes;
 #        dictentry *entries;
-#        lookup_function; # one of the four possible functions for different
+#        lookup_function_no; # one of the four possible functions for different
 #                         # size dicts
 #        (Function DICTKEY, DICTKEY -> bool) *fnkeyeq;
 #        (Function DICTKEY -> int) *fnkeyhash;
@@ -40,6 +40,29 @@
 #
 #
 
+def ll_call_lookup_function(d, key, hash, flag):
+    DICT = lltype.typeOf(d).TO
+    if IS_64BIT:
+        fun = d.lookup_function_no
+        if fun == FUNC_BYTE:
+            return DICT.lookup_family.byte_lookup_function(d, key, hash, flag)
+        elif fun == FUNC_SHORT:
+            return DICT.lookup_family.short_lookup_function(d, key, hash, flag)
+        elif fun == FUNC_INT:
+            return DICT.lookup_family.int_lookup_function(d, key, hash, flag)
+        elif fun == FUNC_LONG:
+            return DICT.lookup_family.long_lookup_function(d, key, hash, flag)
+        assert False
+    else:
+        fun = d.lookup_function_no
+        if fun == FUNC_BYTE:
+            return DICT.lookup_family.byte_lookup_function(d, key, hash, flag)
+        elif fun == FUNC_SHORT:
+            return DICT.lookup_family.short_lookup_function(d, key, hash, flag)
+        elif fun == FUNC_LONG:
+            return DICT.lookup_family.long_lookup_function(d, key, hash, flag)
+        assert False
+
 def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
                 ll_fasthash_function=None, ll_hash_function=None,
                 ll_eq_function=None, method_cache={},
@@ -100,15 +123,11 @@
     DICTENTRY = lltype.Struct("dictentry", *entryfields)
     DICTENTRYARRAY = lltype.GcArray(DICTENTRY,
                                     adtmeths=entrymeths)
-    LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY,
-                                              lltype.Signed, lltype.Signed],
-                                             lltype.Signed))
-
     fields =          [ ("num_items", lltype.Signed),
                         ("num_used_items", lltype.Signed),
                         ("resize_counter", lltype.Signed),
                         ("indexes", llmemory.GCREF),
-                        ("lookup_function", LOOKUP_FUNC),
+                        ("lookup_function_no", lltype.Signed),
                         ("entries", lltype.Ptr(DICTENTRYARRAY)) ]
     if get_custom_eq_hash is not None:
         r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash()
@@ -135,6 +154,7 @@
             }
     adtmeths['KEY']   = DICTKEY
     adtmeths['VALUE'] = DICTVALUE
+    adtmeths['lookup_function'] = lltype.staticAdtMethod(ll_call_lookup_function)
     adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
 
     family = LookupFamily()
@@ -145,10 +165,16 @@
 
     family.empty_array = DICTENTRYARRAY.allocate(0)
     if setup_lookup_funcs:
-        _setup_lookup_funcs(LOOKUP_FUNC, DICT, rtyper, family)
+        _setup_lookup_funcs(DICT, rtyper, family)
     return DICT
 
-def _setup_lookup_funcs(LOOKUP_FUNC, DICT, rtyper, family):
+def _setup_lookup_funcs(DICT, rtyper, family):
+    DICTKEY = DICT.entries.TO.OF.key
+    LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY,
+                                              lltype.Signed, lltype.Signed],
+                                              lltype.Signed))
+
+
     STORECLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
                                                   lltype.Signed,
                                                   lltype.Signed],
@@ -239,10 +265,8 @@
 
     def _setup_repr_final(self):
         if not self.finalized:
-            LOOKUP_FUNC = self.lowleveltype.TO.lookup_function
             family = self.lowleveltype.TO.lookup_family
-            _setup_lookup_funcs(LOOKUP_FUNC, self.lowleveltype.TO, self.rtyper,
-                                family)
+            _setup_lookup_funcs(self.lowleveltype.TO, self.rtyper, family)
             self.finalized = True
 
 
@@ -432,40 +456,43 @@
 
 IS_64BIT = sys.maxint != 2 ** 31 - 1
 
+if IS_64BIT:
+    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG = range(4)
+else:
+    FUNC_BYTE, FUNC_SHORT, FUNC_LONG = range(3)
+
 def ll_malloc_indexes_and_choose_lookup(d, n):
-    DICT = lltype.typeOf(d).TO
     if n <= 256:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_BYTE.TO, n,
                                                          zero=True))
-        d.lookup_function = DICT.lookup_family.byte_lookup_function
+        d.lookup_function_no = FUNC_BYTE
     elif n <= 65536:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_SHORT.TO, n,
                                                          zero=True))
-        d.lookup_function = DICT.lookup_family.short_lookup_function
+        d.lookup_function_no = FUNC_SHORT
     elif IS_64BIT and n <= 2 ** 32:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_INT.TO, n,
                                                          zero=True))
-        d.lookup_function = DICT.lookup_family.int_lookup_function
+        d.lookup_function_no = FUNC_INT
     else:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_LONG.TO, n,
                                                          zero=True))
-        d.lookup_function = DICT.lookup_family.long_lookup_function
-ll_malloc_indexes_and_choose_lookup._always_inline_ = True
+        d.lookup_function_no = FUNC_LONG
 
 def ll_pick_insert_clean_function(d):
     DICT = lltype.typeOf(d).TO
-    if d.lookup_function == DICT.lookup_family.byte_lookup_function:
+    if d.lookup_function_no == FUNC_BYTE:
         return DICT.lookup_family.byte_insert_clean_function
-    if d.lookup_function == DICT.lookup_family.short_lookup_function:
+    if d.lookup_function_no == FUNC_SHORT:
         return DICT.lookup_family.short_insert_clean_function
     if IS_64BIT:
-        if d.lookup_function == DICT.lookup_family.int_lookup_function:
+        if d.lookup_function_no == FUNC_INT:
             return DICT.lookup_family.int_insert_clean_function
-    if d.lookup_function == DICT.lookup_family.long_lookup_function:
+    if d.lookup_function_no == FUNC_LONG:
         return DICT.lookup_family.long_insert_clean_function
     assert False
 

From noreply at buildbot.pypy.org  Thu Oct 24 18:31:21 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 18:31:21 +0200 (CEST)
Subject: [pypy-commit] cffi default: Strip the C line of whitespace in error
	reporting.
Message-ID: <20131024163121.176C21D238E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1380:e90d476329dd
Date: 2013-10-24 18:31 +0200
http://bitbucket.org/cffi/cffi/changeset/e90d476329dd/

Log:	Strip the C line of whitespace in error reporting.

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -142,7 +142,7 @@
                 if 1 <= linenum <= len(csourcelines):
                     line = csourcelines[linenum-1]
         if line:
-            msg = 'cannot parse "%s"\n%s' % (line, msg)
+            msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
             msg = 'parse error\n%s' % (msg,)
         raise api.CDefError(msg)
diff --git a/testing/test_parsing.py b/testing/test_parsing.py
--- a/testing/test_parsing.py
+++ b/testing/test_parsing.py
@@ -190,7 +190,7 @@
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert re.match(r'cannot parse " x y z "\n:\d+:', str(e.value))
+    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
 
 def test_cannot_declare_enum_later():
     ffi = FFI()

From noreply at buildbot.pypy.org  Thu Oct 24 18:51:56 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 18:51:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: add passing test
Message-ID: <20131024165156.2EAC11D2336@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67574:23b6e00c0040
Date: 2013-10-24 18:51 +0200
http://bitbucket.org/pypy/pypy/changeset/23b6e00c0040/

Log:	add passing test

diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1078,6 +1078,13 @@
         res = self.interpret(func, [42])
         assert res == 42
 
+    def test_dict_with_empty_tuple_key(self):
+        def func(i):
+            d = {(): i}
+            return d[()]
+        res = self.interpret(func, [42])
+        assert res == 42
+
 
 class TestStress:
 

From noreply at buildbot.pypy.org  Thu Oct 24 19:46:51 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 24 Oct 2013 19:46:51 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: fix for obscure annotation/jitting
	nonsense
Message-ID: <20131024174651.B6D701C135D@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67575:df2698d1de51
Date: 2013-10-24 10:45 -0700
http://bitbucket.org/pypy/pypy/changeset/df2698d1de51/

Log:	fix for obscure annotation/jitting nonsense

diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -165,7 +165,7 @@
 
     @jit.unroll_safe
     def init_cells(self):
-        if self.cells is None:
+        if not self.cells:
             return
         args_to_copy = self.pycode._args_as_cellvars
         for i in range(len(args_to_copy)):

From noreply at buildbot.pypy.org  Thu Oct 24 19:48:52 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 24 Oct 2013 19:48:52 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: merged default in
Message-ID: <20131024174852.7BA941C135D@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67576:470793d5f2a6
Date: 2013-10-24 10:48 -0700
http://bitbucket.org/pypy/pypy/changeset/470793d5f2a6/

Log:	merged default in

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -283,10 +283,18 @@
         return self.ctype.iter(self)
 
     def unpackiterable_int(self, space):
-        return self.ctype.aslist_int(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_int_items(self)
+        return None
 
     def unpackiterable_float(self, space):
-        return self.ctype.aslist_float(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_float_items(self)
+        return None
 
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -105,26 +105,6 @@
     def iter(self, cdata):
         return W_CDataIter(self.space, self.ctitem, cdata)
 
-    def aslist_int(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_long():
-            res = []
-            buf = rffi.cast(rffi.LONGP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
-    def aslist_float(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_double():
-            res = []
-            buf = rffi.cast(rffi.DOUBLEP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
     def get_vararg_type(self):
         return self.ctptr
 
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -43,10 +43,13 @@
     def is_unichar_ptr_or_array(self):
         return False
 
-    def is_long(self):
-        return False
+    def unpack_list_of_int_items(self, cdata):
+        return None
 
-    def is_double(self):
+    def unpack_list_of_float_items(self, cdata):
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
         return False
 
     def newp(self, w_init):
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -2,6 +2,7 @@
 Primitives.
 """
 
+import sys
 from pypy.interpreter.error import operationerrfmt
 
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
@@ -169,9 +170,9 @@
             sh = self.size * 8
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
-
-    def is_long(self):
-        return self.size == rffi.sizeof(lltype.Signed)
+        else:
+            self.vmin = r_uint(0)
+            self.vrangemax = r_uint(-1)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -204,6 +205,35 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_signed_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.LONG):
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            if self.size == rffi.sizeof(rffi.LONG): # fastest path
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.LONGP, cdata)
+                copy_list_to_raw_array(int_list, cdata)
+            else:
+                overflowed = misc.pack_list_to_raw_array_bounds(
+                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                if overflowed != 0:
+                    self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
     _attrs_            = ['value_fits_long', 'value_fits_ulong', 'vrangemax']
@@ -216,6 +246,8 @@
         self.value_fits_ulong = self.size <= rffi.sizeof(lltype.Unsigned)
         if self.value_fits_long:
             self.vrangemax = self._compute_vrange_max()
+        else:
+            self.vrangemax = r_uint(sys.maxint)
 
     def _compute_vrange_max(self):
         sh = self.size * 8
@@ -255,6 +287,24 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_unsigned_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata,
+                                                     self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            overflowed = misc.pack_list_to_raw_array_bounds(
+                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            if overflowed != 0:
+                self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
     _attrs_ = []
@@ -276,9 +326,6 @@
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
 
-    def is_double(self):
-        return self.size == rffi.sizeof(lltype.Float)
-
     def cast(self, w_ob):
         space = self.space
         if isinstance(w_ob, cdataobj.W_CData):
@@ -318,6 +365,34 @@
         value = space.float_w(space.float(w_ob))
         misc.write_raw_float_data(cdata, value, self.size)
 
+    def unpack_list_of_float_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.DOUBLE):
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.size == rffi.sizeof(rffi.FLOAT):
+            res = [0.0] * w_cdata.get_array_length()
+            misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.DOUBLEP, cdata)
+                copy_list_to_raw_array(float_list, cdata)
+                return True
+            elif self.size == rffi.sizeof(rffi.FLOAT):
+                misc.pack_float_list_to_raw_array(float_list, cdata,
+                                                  rffi.FLOAT, rffi.FLOATP)
+                return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveLongDouble(W_CTypePrimitiveFloat):
     _attrs_ = []
@@ -371,3 +446,15 @@
         else:
             value = space.float_w(space.float(w_ob))
             self._to_longdouble_and_write(value, cdata)
+
+    # Cannot have unpack_list_of_float_items() here:
+    # 'list(array-of-longdouble)' returns a list of cdata objects,
+    # not a list of floats.
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            misc.pack_float_list_to_raw_array(float_list, cdata,
+                                             rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -42,12 +42,6 @@
     def is_char_or_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveCharOrUniChar)
 
-    def aslist_int(self, cdata):
-        return None
-
-    def aslist_float(self, cdata):
-        return None
-
     def cast(self, w_ob):
         # cast to a pointer, to a funcptr, or to an array.
         # Note that casting to an array is an extension to the C language,
@@ -64,24 +58,10 @@
             value = rffi.cast(rffi.CCHARP, value)
         return cdataobj.W_CData(space, value, self)
 
-    def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
-        from rpython.rlib.rarray import copy_list_to_raw_array
-        int_list = self.space.listview_int(w_ob)
-        float_list = self.space.listview_float(w_ob)
+    def _convert_array_from_listview(self, cdata, w_ob):
+        if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
+            return
         #
-        if self.ctitem.is_long() and int_list is not None:
-            cdata = rffi.cast(rffi.LONGP, cdata)
-            copy_list_to_raw_array(int_list, cdata)
-            return True
-        #
-        if self.ctitem.is_double() and float_list is not None:
-            cdata = rffi.cast(rffi.DOUBLEP, cdata)
-            copy_list_to_raw_array(float_list, cdata)
-            return True
-        #
-        return False
-
-    def _convert_array_from_listview(self, cdata, w_ob):
         space = self.space
         lst_w = space.listview(w_ob)
         if self.length >= 0 and len(lst_w) > self.length:
@@ -95,11 +75,6 @@
 
     def convert_array_from_object(self, cdata, w_ob):
         space = self.space
-        if self._convert_array_from_list_strategy_maybe(cdata, w_ob):
-            # the fast path worked, we are done now
-            return
-        #
-        # continue with the slow path
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             self._convert_array_from_listview(cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -315,3 +315,47 @@
             _raw_memclear_tp(TP, TPP, dest)
             return
     raise NotImplementedError("bad clear size")
+
+# ____________________________________________________________
+
+def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) - vmin > vrangemax:
+                    return x      # overflow
+                ptr[i] = rffi.cast(TP, x)
+            return 0
+    raise NotImplementedError("bad integer size")
+
+ at specialize.arg(2)
+def pack_float_list_to_raw_array(float_list, target, TP, TPP):
+    target = rffi.cast(TPP, target)
+    for i in range(len(float_list)):
+        x = float_list[i]
+        target[i] = rffi.cast(TP, x)
+
+def unpack_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_unsigned_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_unsigned_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_cfloat_list_from_raw_array(float_list, source):
+    ptr = rffi.cast(rffi.FLOATP, source)
+    for i in range(len(float_list)):
+        float_list[i] = rffi.cast(lltype.Float, ptr[i])
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -1,18 +1,19 @@
-# side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule()
+# side-effect: FORMAT_LONGDOUBLE must be built before the first test
 from pypy.module._cffi_backend import misc
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeobj import W_CType
+
 
 class AppTest_fast_path_from_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
 
     def setup_method(self, meth):
-        def forbidden(self, *args):
+        def forbidden(*args):
             assert False, 'The slow path is forbidden'
-        self._original = W_CTypePtrOrArray._convert_array_from_listview.im_func
-        W_CTypePtrOrArray._convert_array_from_listview = forbidden
+        self._original = W_CType.pack_list_of_items.im_func
+        W_CType.pack_list_of_items = forbidden
 
     def teardown_method(self, meth):
-        W_CTypePtrOrArray._convert_array_from_listview = self._original
+        W_CType.pack_list_of_items = self._original
 
     def test_fast_init_from_list(self):
         import _cffi_backend
@@ -34,6 +35,101 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
+    def test_fast_init_short_from_list(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, None)
+        buf = _cffi_backend.newp(SHORT_ARRAY, [1, -2, 3])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [40000])
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [-40000])
+
+    def test_fast_init_longlong_from_list(self):
+        if type(2 ** 50) is long:
+            large_int = 2 ** 30
+        else:
+            large_int = 2 ** 50
+        import _cffi_backend
+        LONGLONG = _cffi_backend.new_primitive_type('long long')
+        P_LONGLONG = _cffi_backend.new_pointer_type(LONGLONG)
+        LONGLONG_ARRAY = _cffi_backend.new_array_type(P_LONGLONG, None)
+        buf = _cffi_backend.newp(LONGLONG_ARRAY, [1, -2, 3, large_int])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        assert buf[3] == large_int
+
+    def test_fast_init_ushort_from_list(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, None)
+        buf = _cffi_backend.newp(USHORT_ARRAY, [1, 2, 40000])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == 40000
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [70000])
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [-1])
+
+    def test_fast_init_ulong_from_list(self):
+        import sys
+        import _cffi_backend
+        ULONG = _cffi_backend.new_primitive_type('unsigned long')
+        P_ULONG = _cffi_backend.new_pointer_type(ULONG)
+        ULONG_ARRAY = _cffi_backend.new_array_type(P_ULONG, None)
+        buf = _cffi_backend.newp(ULONG_ARRAY, [1, 2, sys.maxint])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == sys.maxint
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-1])
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-sys.maxint])
+
+    def test_fast_init_cfloat_from_list(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, None)
+        buf = _cffi_backend.newp(FLOAT_ARRAY, [1.25, -3.5])
+        assert buf[0] == 1.25
+        assert buf[1] == -3.5
+
+    def test_fast_init_clongdouble_from_list(self):
+        import _cffi_backend
+        LONGDOUBLE = _cffi_backend.new_primitive_type('long double')
+        P_LONGDOUBLE = _cffi_backend.new_pointer_type(LONGDOUBLE)
+        LONGDOUBLE_ARRAY = _cffi_backend.new_array_type(P_LONGDOUBLE, None)
+        buf = _cffi_backend.newp(LONGDOUBLE_ARRAY, [1.25, -3.5])
+        assert float(buf[0]) == 1.25
+        assert float(buf[1]) == -3.5
+
+    def test_fast_init_bool_from_list(self):
+        import _cffi_backend
+        BOOL = _cffi_backend.new_primitive_type('_Bool')
+        P_BOOL = _cffi_backend.new_pointer_type(BOOL)
+        BOOL_ARRAY = _cffi_backend.new_array_type(P_BOOL, None)
+        buf = _cffi_backend.newp(BOOL_ARRAY, [1, 0])
+        assert buf[0] == 1
+        assert buf[1] == 0
+        assert type(buf[1]) is int
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [2])
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [-1])
+
+
+class AppTest_fast_path_bug(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def test_bug_not_list_or_tuple(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY_2 = _cffi_backend.new_array_type(P_LONG, 2)
+        P_LONG_ARRAY_2 = _cffi_backend.new_pointer_type(LONG_ARRAY_2)
+        LONG_ARRAY_ARRAY = _cffi_backend.new_array_type(P_LONG_ARRAY_2, None)
+        raises(TypeError, _cffi_backend.newp, LONG_ARRAY_ARRAY, [set([4, 5])])
+
 
 class AppTest_fast_path_to_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
@@ -54,12 +150,38 @@
         self._original = original
         rarray.populate_list_from_raw_array = populate_list_from_raw_array
         #
+        original2 = misc.unpack_list_from_raw_array
+        def unpack_list_from_raw_array(*args):
+            self.count += 1
+            return original2(*args)
+        self._original2 = original2
+        misc.unpack_list_from_raw_array = unpack_list_from_raw_array
+        #
+        original3 = misc.unpack_cfloat_list_from_raw_array
+        def unpack_cfloat_list_from_raw_array(*args):
+            self.count += 1
+            return original3(*args)
+        self._original3 = original3
+        misc.unpack_cfloat_list_from_raw_array = (
+            unpack_cfloat_list_from_raw_array)
+        #
+        original4 = misc.unpack_unsigned_list_from_raw_array
+        def unpack_unsigned_list_from_raw_array(*args):
+            self.count += 1
+            return original4(*args)
+        self._original4 = original4
+        misc.unpack_unsigned_list_from_raw_array = (
+            unpack_unsigned_list_from_raw_array)
+        #
         self.w_runappdirect = self.space.wrap(self.runappdirect)
 
 
     def teardown_method(self, meth):
         from rpython.rlib import rarray
         rarray.populate_list_from_raw_array = self._original
+        misc.unpack_list_from_raw_array = self._original2
+        misc.unpack_cfloat_list_from_raw_array = self._original3
+        misc.unpack_unsigned_list_from_raw_array = self._original4
 
     def test_list_int(self):
         import _cffi_backend
@@ -84,6 +206,14 @@
         pbuf = _cffi_backend.cast(P_LONG, buf)
         raises(TypeError, "list(pbuf)")
 
+    def test_bug(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        five = _cffi_backend.cast(LONG, 5)
+        raises(TypeError, list, five)
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
+        raises(TypeError, list, five_and_a_half)
 
     def test_list_float(self):
         import _cffi_backend
@@ -98,3 +228,45 @@
         assert lst == [1.1, 2.2, 3.3]
         if not self.runappdirect:
             assert self.get_count() == 1
+
+    def test_list_short(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, 3)
+        buf = _cffi_backend.newp(SHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_ushort(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, 3)
+        buf = _cffi_backend.newp(USHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 50505
+        lst = list(buf)
+        assert lst == [1, 2, 50505]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_cfloat(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, 3)
+        buf = _cffi_backend.newp(FLOAT_ARRAY)
+        buf[0] = 1.25
+        buf[1] = -2.5
+        buf[2] = 3.75
+        lst = list(buf)
+        assert lst == [1.25, -2.5, 3.75]
+        if not self.runappdirect:
+            assert self.get_count() == 1
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -140,7 +140,7 @@
         ltime = rctime.localtime()
         assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
         try:
-            assert rctime.asctime((12345,) + (0,) * 8).split()[-1] == '12345'
+            rctime.asctime((12345,) + (0,) * 8)  # assert this doesn't crash
         except ValueError:
             pass  # some OS (ie POSIXes besides Linux) reject year > 9999
 
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -11,6 +11,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
+from rpython.translator.backendopt.graphanalyze import DependencyTracker
 
 
 class CallControl(object):
@@ -35,6 +36,7 @@
         #
         for index, jd in enumerate(jitdrivers_sd):
             jd.index = index
+        self.seen = DependencyTracker(self.readwrite_analyzer)
 
     def find_all_graphs(self, policy):
         try:
@@ -231,8 +233,8 @@
                 extraeffect = EffectInfo.EF_CANNOT_RAISE
         #
         effectinfo = effectinfo_from_writeanalyze(
-            self.readwrite_analyzer.analyze(op), self.cpu, extraeffect,
-            oopspecindex, can_invalidate, call_release_gil_target,
+            self.readwrite_analyzer.analyze(op, self.seen), self.cpu,
+            extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
         )
         #
         assert effectinfo is not None
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -82,6 +82,7 @@
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
         elif (opnum != rop.GETFIELD_GC and
+              opnum != rop.GETFIELD_GC_PURE and
               opnum != rop.MARK_OPAQUE_PTR and
               opnum != rop.PTR_EQ and
               opnum != rop.PTR_NE and
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3360,21 +3360,28 @@
         self.check_resops(call=0, getfield_gc=0)
 
     def test_isvirtual_call_assembler(self):
-        driver = JitDriver(greens = ['code'], reds = ['n'])
+        driver = JitDriver(greens = ['code'], reds = ['n', 's'])
 
         @look_inside_iff(lambda t1, t2: isvirtual(t1))
         def g(t1, t2):
             return t1[0] == t2[0]
 
+        def create(n):
+            return (1, 2, n)
+        create._dont_inline_ = True
+
         def f(code, n):
+            s = 0
             while n > 0:
-                driver.can_enter_jit(code=code, n=n)
-                driver.jit_merge_point(code=code, n=n)
-                t = (1, 2, n)
+                driver.can_enter_jit(code=code, n=n, s=s)
+                driver.jit_merge_point(code=code, n=n, s=s)
+                t = create(n)
                 if code:
                     f(0, 3)
+                s += t[2]
                 g(t, (1, 2, n))
                 n -= 1
+            return s
 
         self.meta_interp(f, [1, 10], inline=True)
         self.check_resops(call=0, call_may_force=0, call_assembler=2)
diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py
--- a/rpython/rlib/rstack.py
+++ b/rpython/rlib/rstack.py
@@ -67,6 +67,7 @@
     # Else call the slow path
     stack_check_slowpath(current)
 stack_check._always_inline_ = True
+stack_check._dont_insert_stackcheck_ = True
 
 @rgc.no_collect
 def stack_check_slowpath(current):
@@ -74,3 +75,4 @@
         from rpython.rlib.rstackovf import _StackOverflow
         raise _StackOverflow
 stack_check_slowpath._dont_inline_ = True
+stack_check_slowpath._dont_insert_stackcheck_ = True
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -97,8 +97,16 @@
     def __eq__(self, other):
         if isinstance(other, Typedef):
             return other.__eq__(self)
-        return self.__class__ is other.__class__ and (
-            self is other or safe_equal(self.__dict__, other.__dict__))
+        if self.__class__ is other.__class__:
+            if self is other:
+                return True
+            try:
+                if hash(self) != hash(other):
+                    return False
+            except TypeError:
+                pass # too bad, we can't use a fastpath here
+            return safe_equal(self.__dict__, other.__dict__)
+        return False
 
     def __ne__(self, other):
         return not (self == other)
@@ -227,6 +235,9 @@
         self.OF = OF
         self.c_name = c_name
 
+    def __hash__(self):
+        return hash(self.OF)
+
     def __repr__(self):
         return '<Typedef "%s" of %r>' % (self.c_name, self.OF)
 
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1078,6 +1078,13 @@
         res = self.interpret(func, [42])
         assert res == 42
 
+    def test_dict_with_empty_tuple_key(self):
+        def func(i):
+            d = {(): i}
+            return d[()]
+        res = self.interpret(func, [42])
+        assert res == 42
+
 
 class TestStress:
 
diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -4,6 +4,7 @@
 top_set = object()
 empty_set = frozenset()
 
+CUTOFF = 1000
 
 class WriteAnalyzer(graphanalyze.GraphAnalyzer):
     def bottom_result(self):
@@ -21,6 +22,8 @@
     def add_to_result(self, result, other):
         if other is top_set:
             return top_set
+        if len(other) + len(result) > CUTOFF:
+            return top_set
         result.update(other)
         return result
 
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -213,6 +213,10 @@
     insert_in = set()
     block2graph = {}
     for caller in translator.graphs:
+        pyobj = getattr(caller, 'func', None)
+        if pyobj is not None:
+            if getattr(pyobj, '_dont_insert_stackcheck_', False):
+                continue
         for block, callee in find_calls_from(translator, caller):
             if getattr(getattr(callee, 'func', None),
                        'insert_stack_check_here', False):
@@ -269,4 +273,4 @@
     transform_dead_op_vars(ann, block_subset)
     if ann.translator:
         checkgraphs(ann, block_subset)
- 
+

From noreply at buildbot.pypy.org  Thu Oct 24 19:57:28 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 19:57:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix the tests
Message-ID: <20131024175728.3312F1C1380@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67577:05f2c0f37596
Date: 2013-10-24 19:56 +0200
http://bitbucket.org/pypy/pypy/changeset/05f2c0f37596/

Log:	fix the tests

diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -33,10 +33,12 @@
             self.virtualizable_analyzer = VirtualizableAnalyzer(translator)
             self.quasiimmut_analyzer = QuasiImmutAnalyzer(translator)
             self.randomeffects_analyzer = RandomEffectsAnalyzer(translator)
+            self.seen = DependencyTracker(self.readwrite_analyzer)
+        else:
+            self.seen = None
         #
         for index, jd in enumerate(jitdrivers_sd):
             jd.index = index
-        self.seen = DependencyTracker(self.readwrite_analyzer)
 
     def find_all_graphs(self, policy):
         try:

From noreply at buildbot.pypy.org  Thu Oct 24 20:05:45 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 24 Oct 2013 20:05:45 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: yet another attempt at
	fixing indirect calls
Message-ID: <20131024180545.316F11C1380@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67578:cbca3da21bd4
Date: 2013-10-24 20:05 +0200
http://bitbucket.org/pypy/pypy/changeset/cbca3da21bd4/

Log:	yet another attempt at fixing indirect calls

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -42,26 +42,16 @@
 
 def ll_call_lookup_function(d, key, hash, flag):
     DICT = lltype.typeOf(d).TO
-    if IS_64BIT:
-        fun = d.lookup_function_no
-        if fun == FUNC_BYTE:
-            return DICT.lookup_family.byte_lookup_function(d, key, hash, flag)
-        elif fun == FUNC_SHORT:
-            return DICT.lookup_family.short_lookup_function(d, key, hash, flag)
-        elif fun == FUNC_INT:
-            return DICT.lookup_family.int_lookup_function(d, key, hash, flag)
-        elif fun == FUNC_LONG:
-            return DICT.lookup_family.long_lookup_function(d, key, hash, flag)
-        assert False
-    else:
-        fun = d.lookup_function_no
-        if fun == FUNC_BYTE:
-            return DICT.lookup_family.byte_lookup_function(d, key, hash, flag)
-        elif fun == FUNC_SHORT:
-            return DICT.lookup_family.short_lookup_function(d, key, hash, flag)
-        elif fun == FUNC_LONG:
-            return DICT.lookup_family.long_lookup_function(d, key, hash, flag)
-        assert False
+    fun = d.lookup_function_no
+    if fun == FUNC_BYTE:
+        return DICT.lookup_family.byte_lookup_function(d, key, hash, flag)
+    elif fun == FUNC_SHORT:
+        return DICT.lookup_family.short_lookup_function(d, key, hash, flag)
+    elif IS_64BIT and fun == FUNC_INT:
+        return DICT.lookup_family.int_lookup_function(d, key, hash, flag)
+    elif fun == FUNC_LONG:
+        return DICT.lookup_family.long_lookup_function(d, key, hash, flag)
+    assert False
 
 def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
                 ll_fasthash_function=None, ll_hash_function=None,
@@ -483,18 +473,18 @@
                                                          zero=True))
         d.lookup_function_no = FUNC_LONG
 
-def ll_pick_insert_clean_function(d):
+def ll_call_insert_clean_function(d, hash, i):
     DICT = lltype.typeOf(d).TO
     if d.lookup_function_no == FUNC_BYTE:
-        return DICT.lookup_family.byte_insert_clean_function
-    if d.lookup_function_no == FUNC_SHORT:
-        return DICT.lookup_family.short_insert_clean_function
-    if IS_64BIT:
-        if d.lookup_function_no == FUNC_INT:
-            return DICT.lookup_family.int_insert_clean_function
-    if d.lookup_function_no == FUNC_LONG:
-        return DICT.lookup_family.long_insert_clean_function
-    assert False
+        DICT.lookup_family.byte_insert_clean_function(d, hash, i)
+    elif d.lookup_function_no == FUNC_SHORT:
+        DICT.lookup_family.short_insert_clean_function(d, hash, i)
+    elif IS_64BIT and d.lookup_function_no == FUNC_INT:
+        DICT.lookup_family.int_insert_clean_function(d, hash, i)
+    elif d.lookup_function_no == FUNC_LONG:
+        DICT.lookup_family.long_insert_clean_function(d, hash, i)
+    else:
+        assert False
 
 def ll_valid_from_flag(entries, i):
     return entries[i].f_valid
@@ -567,8 +557,7 @@
     else:
         if len(d.entries) == d.num_used_items:
             if ll_dict_grow(d):
-                insertcleanfn = ll_pick_insert_clean_function(d)
-                insertcleanfn(d, hash, d.num_used_items)
+                ll_call_insert_clean_function(d, hash, d.num_used_items)
         entry = d.entries[d.num_used_items]
         entry.key = key
         entry.value = value
@@ -587,8 +576,7 @@
 
 def _ll_dict_insertclean(d, key, value, hash):
     ENTRY = lltype.typeOf(d.entries).TO.OF
-    insertcleanfn = ll_pick_insert_clean_function(d)
-    insertcleanfn(d, hash, d.num_used_items)
+    ll_call_insert_clean_function(d, hash, d.num_used_items)
     entry = d.entries[d.num_used_items]
     entry.key = key
     entry.value = value
@@ -739,13 +727,12 @@
     d.resize_counter = new_size * 2 - d.num_items * 3
     assert d.resize_counter > 0
     #
-    insertcleanfn = ll_pick_insert_clean_function(d)
     entries = d.entries
     i = 0
     while i < d.num_used_items:
         if entries.valid(i):
             hash = entries.hash(i)
-            insertcleanfn(d, hash, i)
+            ll_call_insert_clean_function(d, hash, i)
         i += 1
     #old_entries.delete() XXXX!
 

From noreply at buildbot.pypy.org  Thu Oct 24 20:12:03 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 20:12:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a test: this misusage of
 virtualizable arrays should be forbidden.
Message-ID: <20131024181203.79B471C00D8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67579:f443637ade53
Date: 2013-10-24 19:56 +0200
http://bitbucket.org/pypy/pypy/changeset/f443637ade53/

Log:	Add a test: this misusage of virtualizable arrays should be
	forbidden. In some cases it would generate bogus flattened jitcodes
	that fail to assemble.

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -93,6 +93,8 @@
         block.exitswitch = renamings.get(block.exitswitch, block.exitswitch)
         self.follow_constant_exit(block)
         self.optimize_goto_if_not(block)
+        if isinstance(block.exitswitch, tuple):
+            self._check_no_vable_array(block.exitswitch)
         for link in block.exits:
             self._check_no_vable_array(link.args)
             self._do_renaming_on_link(renamings, link)
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -84,6 +84,8 @@
     def calldescr_canraise(self, calldescr):
         return calldescr is not self._descr_cannot_raise and calldescr.oopspecindex == EffectInfo.OS_NONE
     def get_vinfo(self, VTYPEPTR):
+        if hasattr(VTYPEPTR.TO, 'inst_vlist'):
+            return FakeVInfo()
         return None
 
 class FakeCallControlWithVRefInfo:
@@ -100,6 +102,13 @@
     def calldescr_canraise(self, calldescr):
         return False
 
+class FakeVInfo:
+    static_field_to_extra_box = {}
+    array_fields = {'inst_vlist': '?'}
+    array_field_counter = {'inst_vlist': 0}
+    array_field_descrs = [FakeDescr()]
+    array_descrs = [FakeDescr()]
+
 # ____________________________________________________________
 
 def test_reorder_renaming_list():
@@ -1001,6 +1010,22 @@
             float_return %(result_var)s
         """ % {"result_var": result_var, "tmp_var": tmp_var}, transform=True)
 
+    def test_vable_attribute_list_is_not_None(self):
+        class F:
+            _virtualizable_ = ['vlist[*]']
+            vlist = None
+            def __init__(self, x):
+                self.vlist = [x]
+        def g():
+            return F(42)
+        def f():
+            f = g()
+            if f.vlist is not None:
+                pass
+        e = py.test.raises(AssertionError, self.encoding_test, f, [], "!",
+                           transform=True)
+        assert str(e.value).startswith("A virtualizable array is passed aroun")
+
 
 def check_force_cast(FROM, TO, operations, value):
     """Check that the test is correctly written..."""

From noreply at buildbot.pypy.org  Thu Oct 24 20:12:04 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 20:12:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Make this code compliant with
 "virtualizable arrays are never None".
Message-ID: <20131024181204.95CE91C00D8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67580:d011e042323a
Date: 2013-10-24 20:07 +0200
http://bitbucket.org/pypy/pypy/changeset/d011e042323a/

Log:	Make this code compliant with "virtualizable arrays are never None".

diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -125,8 +125,7 @@
         if len(cellvars) != ncellvars:
             raise OperationError(self.space.w_TypeError,
                                  self.space.wrap("bad cellvars"))
-        if self.cells is not None:
-            self.cells[:ncellvars] = cellvars
+        self.cells[:ncellvars] = cellvars
 
     @jit.dont_look_inside
     def fast2locals(self):
@@ -165,8 +164,6 @@
 
     @jit.unroll_safe
     def init_cells(self):
-        if self.cells is None:
-            return
         args_to_copy = self.pycode._args_as_cellvars
         for i in range(len(args_to_copy)):
             argnum = args_to_copy[i]

From noreply at buildbot.pypy.org  Thu Oct 24 20:12:06 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 24 Oct 2013 20:12:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20131024181206.36F2D1C00D8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67581:643412f149d4
Date: 2013-10-24 20:11 +0200
http://bitbucket.org/pypy/pypy/changeset/643412f149d4/

Log:	merge heads

diff --git a/lib_pypy/numpypy/lib/utils.py b/lib_pypy/numpypy/lib/utils.py
--- a/lib_pypy/numpypy/lib/utils.py
+++ b/lib_pypy/numpypy/lib/utils.py
@@ -21,14 +21,4 @@
         ...
 
     """
-    try:
-        import numpy
-    except:
-        # running from pypy source directory
-        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-        return os.path.join(head, '../include')
-    else:
-        # using installed numpy core headers
-        import numpy.core as core
-        d = os.path.join(os.path.dirname(core.__file__), 'include')
-    return d
+    return os.path.join(os.path.dirname(__file__), '../../../include')
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -283,10 +283,18 @@
         return self.ctype.iter(self)
 
     def unpackiterable_int(self, space):
-        return self.ctype.aslist_int(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_int_items(self)
+        return None
 
     def unpackiterable_float(self, space):
-        return self.ctype.aslist_float(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_float_items(self)
+        return None
 
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -105,26 +105,6 @@
     def iter(self, cdata):
         return W_CDataIter(self.space, self.ctitem, cdata)
 
-    def aslist_int(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_long():
-            res = []
-            buf = rffi.cast(rffi.LONGP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
-    def aslist_float(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_double():
-            res = []
-            buf = rffi.cast(rffi.DOUBLEP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
     def get_vararg_type(self):
         return self.ctptr
 
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -43,10 +43,13 @@
     def is_unichar_ptr_or_array(self):
         return False
 
-    def is_long(self):
-        return False
+    def unpack_list_of_int_items(self, cdata):
+        return None
 
-    def is_double(self):
+    def unpack_list_of_float_items(self, cdata):
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
         return False
 
     def newp(self, w_init):
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -2,6 +2,7 @@
 Primitives.
 """
 
+import sys
 from pypy.interpreter.error import operationerrfmt
 
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
@@ -169,9 +170,9 @@
             sh = self.size * 8
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
-
-    def is_long(self):
-        return self.size == rffi.sizeof(lltype.Signed)
+        else:
+            self.vmin = r_uint(0)
+            self.vrangemax = r_uint(-1)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -204,6 +205,35 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_signed_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.LONG):
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            if self.size == rffi.sizeof(rffi.LONG): # fastest path
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.LONGP, cdata)
+                copy_list_to_raw_array(int_list, cdata)
+            else:
+                overflowed = misc.pack_list_to_raw_array_bounds(
+                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                if overflowed != 0:
+                    self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
     _attrs_            = ['value_fits_long', 'value_fits_ulong', 'vrangemax']
@@ -216,6 +246,8 @@
         self.value_fits_ulong = self.size <= rffi.sizeof(lltype.Unsigned)
         if self.value_fits_long:
             self.vrangemax = self._compute_vrange_max()
+        else:
+            self.vrangemax = r_uint(sys.maxint)
 
     def _compute_vrange_max(self):
         sh = self.size * 8
@@ -255,6 +287,24 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_unsigned_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata,
+                                                     self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            overflowed = misc.pack_list_to_raw_array_bounds(
+                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            if overflowed != 0:
+                self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
     _attrs_ = []
@@ -276,9 +326,6 @@
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
 
-    def is_double(self):
-        return self.size == rffi.sizeof(lltype.Float)
-
     def cast(self, w_ob):
         space = self.space
         if isinstance(w_ob, cdataobj.W_CData):
@@ -318,6 +365,34 @@
         value = space.float_w(space.float(w_ob))
         misc.write_raw_float_data(cdata, value, self.size)
 
+    def unpack_list_of_float_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.DOUBLE):
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.size == rffi.sizeof(rffi.FLOAT):
+            res = [0.0] * w_cdata.get_array_length()
+            misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.DOUBLEP, cdata)
+                copy_list_to_raw_array(float_list, cdata)
+                return True
+            elif self.size == rffi.sizeof(rffi.FLOAT):
+                misc.pack_float_list_to_raw_array(float_list, cdata,
+                                                  rffi.FLOAT, rffi.FLOATP)
+                return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveLongDouble(W_CTypePrimitiveFloat):
     _attrs_ = []
@@ -328,7 +403,6 @@
         return misc.longdouble2str(lvalue)
 
     def cast(self, w_ob):
-        space = self.space
         if (isinstance(w_ob, cdataobj.W_CData) and
                 isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)):
             w_cdata = self.convert_to_object(w_ob._cdata)
@@ -372,3 +446,15 @@
         else:
             value = space.float_w(space.float(w_ob))
             self._to_longdouble_and_write(value, cdata)
+
+    # Cannot have unpack_list_of_float_items() here:
+    # 'list(array-of-longdouble)' returns a list of cdata objects,
+    # not a list of floats.
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            misc.pack_float_list_to_raw_array(float_list, cdata,
+                                             rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -42,12 +42,6 @@
     def is_char_or_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveCharOrUniChar)
 
-    def aslist_int(self, cdata):
-        return None
-
-    def aslist_float(self, cdata):
-        return None
-
     def cast(self, w_ob):
         # cast to a pointer, to a funcptr, or to an array.
         # Note that casting to an array is an extension to the C language,
@@ -64,24 +58,10 @@
             value = rffi.cast(rffi.CCHARP, value)
         return cdataobj.W_CData(space, value, self)
 
-    def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
-        from rpython.rlib.rarray import copy_list_to_raw_array
-        int_list = self.space.listview_int(w_ob)
-        float_list = self.space.listview_float(w_ob)
+    def _convert_array_from_listview(self, cdata, w_ob):
+        if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
+            return
         #
-        if self.ctitem.is_long() and int_list is not None:
-            cdata = rffi.cast(rffi.LONGP, cdata)
-            copy_list_to_raw_array(int_list, cdata)
-            return True
-        #
-        if self.ctitem.is_double() and float_list is not None:
-            cdata = rffi.cast(rffi.DOUBLEP, cdata)
-            copy_list_to_raw_array(float_list, cdata)
-            return True
-        #
-        return False
-
-    def _convert_array_from_listview(self, cdata, w_ob):
         space = self.space
         lst_w = space.listview(w_ob)
         if self.length >= 0 and len(lst_w) > self.length:
@@ -95,11 +75,6 @@
 
     def convert_array_from_object(self, cdata, w_ob):
         space = self.space
-        if self._convert_array_from_list_strategy_maybe(cdata, w_ob):
-            # the fast path worked, we are done now
-            return
-        #
-        # continue with the slow path
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             self._convert_array_from_listview(cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -315,3 +315,47 @@
             _raw_memclear_tp(TP, TPP, dest)
             return
     raise NotImplementedError("bad clear size")
+
+# ____________________________________________________________
+
+def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) - vmin > vrangemax:
+                    return x      # overflow
+                ptr[i] = rffi.cast(TP, x)
+            return 0
+    raise NotImplementedError("bad integer size")
+
+ at specialize.arg(2)
+def pack_float_list_to_raw_array(float_list, target, TP, TPP):
+    target = rffi.cast(TPP, target)
+    for i in range(len(float_list)):
+        x = float_list[i]
+        target[i] = rffi.cast(TP, x)
+
+def unpack_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_unsigned_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_unsigned_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_cfloat_list_from_raw_array(float_list, source):
+    ptr = rffi.cast(rffi.FLOATP, source)
+    for i in range(len(float_list)):
+        float_list[i] = rffi.cast(lltype.Float, ptr[i])
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -117,13 +117,17 @@
 
 SF_MSVC_BITFIELDS = 1
 SF_GCC_ARM_BITFIELDS = 2
+SF_GCC_BIG_ENDIAN = 4
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
-elif rffi_platform.getdefined('__arm__', ''):
-    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
-    DEFAULT_SFLAGS = 0
+    if rffi_platform.getdefined('__arm__', ''):
+        DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
+    else:
+        DEFAULT_SFLAGS = 0
+    if sys.byteorder == 'big':
+        DEFAULT_SFLAGS |= SF_GCC_BIG_ENDIAN
 
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
@@ -325,6 +329,9 @@
                     prev_bitfield_free -= fbitsize
                     field_offset_bytes = boffset / 8 - ftype.size
 
+                if sflags & SF_GCC_BIG_ENDIAN:
+                    bitshift = 8 * ftype.size - fbitsize- bitshift
+
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
                                            bitshift, fbitsize)
                 fields_list.append(fld)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2949,8 +2949,6 @@
     _test_bitfield_details(flag=2)
 
 def test_bitfield_as_big_endian():
-    if '__pypy__' in sys.builtin_module_names:
-        py.test.skip("no big endian machine supported on pypy for now")
     _test_bitfield_details(flag=4)
 
 
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -1,18 +1,19 @@
-# side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule()
+# side-effect: FORMAT_LONGDOUBLE must be built before the first test
 from pypy.module._cffi_backend import misc
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeobj import W_CType
+
 
 class AppTest_fast_path_from_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
 
     def setup_method(self, meth):
-        def forbidden(self, *args):
+        def forbidden(*args):
             assert False, 'The slow path is forbidden'
-        self._original = W_CTypePtrOrArray._convert_array_from_listview.im_func
-        W_CTypePtrOrArray._convert_array_from_listview = forbidden
+        self._original = W_CType.pack_list_of_items.im_func
+        W_CType.pack_list_of_items = forbidden
 
     def teardown_method(self, meth):
-        W_CTypePtrOrArray._convert_array_from_listview = self._original
+        W_CType.pack_list_of_items = self._original
 
     def test_fast_init_from_list(self):
         import _cffi_backend
@@ -34,6 +35,101 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
+    def test_fast_init_short_from_list(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, None)
+        buf = _cffi_backend.newp(SHORT_ARRAY, [1, -2, 3])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [40000])
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [-40000])
+
+    def test_fast_init_longlong_from_list(self):
+        if type(2 ** 50) is long:
+            large_int = 2 ** 30
+        else:
+            large_int = 2 ** 50
+        import _cffi_backend
+        LONGLONG = _cffi_backend.new_primitive_type('long long')
+        P_LONGLONG = _cffi_backend.new_pointer_type(LONGLONG)
+        LONGLONG_ARRAY = _cffi_backend.new_array_type(P_LONGLONG, None)
+        buf = _cffi_backend.newp(LONGLONG_ARRAY, [1, -2, 3, large_int])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        assert buf[3] == large_int
+
+    def test_fast_init_ushort_from_list(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, None)
+        buf = _cffi_backend.newp(USHORT_ARRAY, [1, 2, 40000])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == 40000
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [70000])
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [-1])
+
+    def test_fast_init_ulong_from_list(self):
+        import sys
+        import _cffi_backend
+        ULONG = _cffi_backend.new_primitive_type('unsigned long')
+        P_ULONG = _cffi_backend.new_pointer_type(ULONG)
+        ULONG_ARRAY = _cffi_backend.new_array_type(P_ULONG, None)
+        buf = _cffi_backend.newp(ULONG_ARRAY, [1, 2, sys.maxint])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == sys.maxint
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-1])
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-sys.maxint])
+
+    def test_fast_init_cfloat_from_list(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, None)
+        buf = _cffi_backend.newp(FLOAT_ARRAY, [1.25, -3.5])
+        assert buf[0] == 1.25
+        assert buf[1] == -3.5
+
+    def test_fast_init_clongdouble_from_list(self):
+        import _cffi_backend
+        LONGDOUBLE = _cffi_backend.new_primitive_type('long double')
+        P_LONGDOUBLE = _cffi_backend.new_pointer_type(LONGDOUBLE)
+        LONGDOUBLE_ARRAY = _cffi_backend.new_array_type(P_LONGDOUBLE, None)
+        buf = _cffi_backend.newp(LONGDOUBLE_ARRAY, [1.25, -3.5])
+        assert float(buf[0]) == 1.25
+        assert float(buf[1]) == -3.5
+
+    def test_fast_init_bool_from_list(self):
+        import _cffi_backend
+        BOOL = _cffi_backend.new_primitive_type('_Bool')
+        P_BOOL = _cffi_backend.new_pointer_type(BOOL)
+        BOOL_ARRAY = _cffi_backend.new_array_type(P_BOOL, None)
+        buf = _cffi_backend.newp(BOOL_ARRAY, [1, 0])
+        assert buf[0] == 1
+        assert buf[1] == 0
+        assert type(buf[1]) is int
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [2])
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [-1])
+
+
+class AppTest_fast_path_bug(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def test_bug_not_list_or_tuple(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY_2 = _cffi_backend.new_array_type(P_LONG, 2)
+        P_LONG_ARRAY_2 = _cffi_backend.new_pointer_type(LONG_ARRAY_2)
+        LONG_ARRAY_ARRAY = _cffi_backend.new_array_type(P_LONG_ARRAY_2, None)
+        raises(TypeError, _cffi_backend.newp, LONG_ARRAY_ARRAY, [set([4, 5])])
+
 
 class AppTest_fast_path_to_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
@@ -54,12 +150,38 @@
         self._original = original
         rarray.populate_list_from_raw_array = populate_list_from_raw_array
         #
+        original2 = misc.unpack_list_from_raw_array
+        def unpack_list_from_raw_array(*args):
+            self.count += 1
+            return original2(*args)
+        self._original2 = original2
+        misc.unpack_list_from_raw_array = unpack_list_from_raw_array
+        #
+        original3 = misc.unpack_cfloat_list_from_raw_array
+        def unpack_cfloat_list_from_raw_array(*args):
+            self.count += 1
+            return original3(*args)
+        self._original3 = original3
+        misc.unpack_cfloat_list_from_raw_array = (
+            unpack_cfloat_list_from_raw_array)
+        #
+        original4 = misc.unpack_unsigned_list_from_raw_array
+        def unpack_unsigned_list_from_raw_array(*args):
+            self.count += 1
+            return original4(*args)
+        self._original4 = original4
+        misc.unpack_unsigned_list_from_raw_array = (
+            unpack_unsigned_list_from_raw_array)
+        #
         self.w_runappdirect = self.space.wrap(self.runappdirect)
 
 
     def teardown_method(self, meth):
         from rpython.rlib import rarray
         rarray.populate_list_from_raw_array = self._original
+        misc.unpack_list_from_raw_array = self._original2
+        misc.unpack_cfloat_list_from_raw_array = self._original3
+        misc.unpack_unsigned_list_from_raw_array = self._original4
 
     def test_list_int(self):
         import _cffi_backend
@@ -84,6 +206,14 @@
         pbuf = _cffi_backend.cast(P_LONG, buf)
         raises(TypeError, "list(pbuf)")
 
+    def test_bug(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        five = _cffi_backend.cast(LONG, 5)
+        raises(TypeError, list, five)
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
+        raises(TypeError, list, five_and_a_half)
 
     def test_list_float(self):
         import _cffi_backend
@@ -98,3 +228,45 @@
         assert lst == [1.1, 2.2, 3.3]
         if not self.runappdirect:
             assert self.get_count() == 1
+
+    def test_list_short(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, 3)
+        buf = _cffi_backend.newp(SHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_ushort(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, 3)
+        buf = _cffi_backend.newp(USHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 50505
+        lst = list(buf)
+        assert lst == [1, 2, 50505]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_cfloat(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, 3)
+        buf = _cffi_backend.newp(FLOAT_ARRAY)
+        buf[0] = 1.25
+        buf[1] = -2.5
+        buf[2] = 3.75
+        lst = list(buf)
+        assert lst == [1.25, -2.5, 3.75]
+        if not self.runappdirect:
+            assert self.get_count() == 1
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -84,6 +84,7 @@
 
     def build_and_convert(self, space, box):
         return self.itemtype.build_and_convert(space, self, box)
+
     def coerce(self, space, w_item):
         return self.itemtype.coerce(space, self, w_item)
 
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -140,7 +140,7 @@
         ltime = rctime.localtime()
         assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
         try:
-            assert rctime.asctime((12345,) + (0,) * 8).split()[-1] == '12345'
+            rctime.asctime((12345,) + (0,) * 8)  # assert this doesn't crash
         except ValueError:
             pass  # some OS (ie POSIXes besides Linux) reject year > 9999
 
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -11,6 +11,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
+from rpython.translator.backendopt.graphanalyze import DependencyTracker
 
 
 class CallControl(object):
@@ -32,6 +33,9 @@
             self.virtualizable_analyzer = VirtualizableAnalyzer(translator)
             self.quasiimmut_analyzer = QuasiImmutAnalyzer(translator)
             self.randomeffects_analyzer = RandomEffectsAnalyzer(translator)
+            self.seen = DependencyTracker(self.readwrite_analyzer)
+        else:
+            self.seen = None
         #
         for index, jd in enumerate(jitdrivers_sd):
             jd.index = index
@@ -231,8 +235,8 @@
                 extraeffect = EffectInfo.EF_CANNOT_RAISE
         #
         effectinfo = effectinfo_from_writeanalyze(
-            self.readwrite_analyzer.analyze(op), self.cpu, extraeffect,
-            oopspecindex, can_invalidate, call_release_gil_target,
+            self.readwrite_analyzer.analyze(op, self.seen), self.cpu,
+            extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
         )
         #
         assert effectinfo is not None
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -82,6 +82,7 @@
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
         elif (opnum != rop.GETFIELD_GC and
+              opnum != rop.GETFIELD_GC_PURE and
               opnum != rop.MARK_OPAQUE_PTR and
               opnum != rop.PTR_EQ and
               opnum != rop.PTR_NE and
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1398,7 +1398,7 @@
                     assembler_call_jd)
             if resbox is not None:
                 self.make_result_of_lastop(resbox)
-            self.metainterp.vable_after_residual_call()
+            self.metainterp.vable_after_residual_call(funcbox)
             self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None)
             if vablebox is not None:
                 self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None)
@@ -2437,7 +2437,7 @@
                 # it by ConstPtr(NULL).
                 self.stop_tracking_virtualref(i)
 
-    def vable_after_residual_call(self):
+    def vable_after_residual_call(self, funcbox):
         vinfo = self.jitdriver_sd.virtualizable_info
         if vinfo is not None:
             virtualizable_box = self.virtualizable_boxes[-1]
@@ -2445,6 +2445,14 @@
             if vinfo.tracing_after_residual_call(virtualizable):
                 # the virtualizable escaped during CALL_MAY_FORCE.
                 self.load_fields_from_virtualizable()
+                target_name = self.staticdata.get_name_from_address(funcbox.getaddr())
+                if target_name:
+                    target_name = "ConstClass(%s)" % target_name
+                else:
+                    target_name = str(funcbox.getaddr())
+                debug_print('vable escaped during a call in %s to %s' % (
+                    self.framestack[-1].jitcode.name, target_name
+                ))
                 raise SwitchToBlackhole(Counters.ABORT_ESCAPE,
                                         raising_exception=True)
                 # ^^^ we set 'raising_exception' to True because we must still
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3360,21 +3360,28 @@
         self.check_resops(call=0, getfield_gc=0)
 
     def test_isvirtual_call_assembler(self):
-        driver = JitDriver(greens = ['code'], reds = ['n'])
+        driver = JitDriver(greens = ['code'], reds = ['n', 's'])
 
         @look_inside_iff(lambda t1, t2: isvirtual(t1))
         def g(t1, t2):
             return t1[0] == t2[0]
 
+        def create(n):
+            return (1, 2, n)
+        create._dont_inline_ = True
+
         def f(code, n):
+            s = 0
             while n > 0:
-                driver.can_enter_jit(code=code, n=n)
-                driver.jit_merge_point(code=code, n=n)
-                t = (1, 2, n)
+                driver.can_enter_jit(code=code, n=n, s=s)
+                driver.jit_merge_point(code=code, n=n, s=s)
+                t = create(n)
                 if code:
                     f(0, 3)
+                s += t[2]
                 g(t, (1, 2, n))
                 n -= 1
+            return s
 
         self.meta_interp(f, [1, 10], inline=True)
         self.check_resops(call=0, call_may_force=0, call_assembler=2)
diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py
--- a/rpython/rlib/rstack.py
+++ b/rpython/rlib/rstack.py
@@ -67,6 +67,7 @@
     # Else call the slow path
     stack_check_slowpath(current)
 stack_check._always_inline_ = True
+stack_check._dont_insert_stackcheck_ = True
 
 @rgc.no_collect
 def stack_check_slowpath(current):
@@ -74,3 +75,4 @@
         from rpython.rlib.rstackovf import _StackOverflow
         raise _StackOverflow
 stack_check_slowpath._dont_inline_ = True
+stack_check_slowpath._dont_insert_stackcheck_ = True
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1078,6 +1078,13 @@
         res = self.interpret(func, [42])
         assert res == 42
 
+    def test_dict_with_empty_tuple_key(self):
+        def func(i):
+            d = {(): i}
+            return d[()]
+        res = self.interpret(func, [42])
+        assert res == 42
+
 
 class TestStress:
 
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -213,6 +213,10 @@
     insert_in = set()
     block2graph = {}
     for caller in translator.graphs:
+        pyobj = getattr(caller, 'func', None)
+        if pyobj is not None:
+            if getattr(pyobj, '_dont_insert_stackcheck_', False):
+                continue
         for block, callee in find_calls_from(translator, caller):
             if getattr(getattr(callee, 'func', None),
                        'insert_stack_check_here', False):
@@ -269,4 +273,4 @@
     transform_dead_op_vars(ann, block_subset)
     if ann.translator:
         checkgraphs(ann, block_subset)
- 
+

From noreply at buildbot.pypy.org  Thu Oct 24 20:12:35 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 24 Oct 2013 20:12:35 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: Backed out changeset 470793d5f2a6
Message-ID: <20131024181235.C8EE71C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67582:3bfebfdea103
Date: 2013-10-24 11:11 -0700
http://bitbucket.org/pypy/pypy/changeset/3bfebfdea103/

Log:	Backed out changeset 470793d5f2a6

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -283,18 +283,10 @@
         return self.ctype.iter(self)
 
     def unpackiterable_int(self, space):
-        from pypy.module._cffi_backend import ctypearray
-        ctype = self.ctype
-        if isinstance(ctype, ctypearray.W_CTypeArray):
-            return ctype.ctitem.unpack_list_of_int_items(self)
-        return None
+        return self.ctype.aslist_int(self)
 
     def unpackiterable_float(self, space):
-        from pypy.module._cffi_backend import ctypearray
-        ctype = self.ctype
-        if isinstance(ctype, ctypearray.W_CTypeArray):
-            return ctype.ctitem.unpack_list_of_float_items(self)
-        return None
+        return self.ctype.aslist_float(self)
 
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -105,6 +105,26 @@
     def iter(self, cdata):
         return W_CDataIter(self.space, self.ctitem, cdata)
 
+    def aslist_int(self, cdata):
+        from rpython.rlib.rarray import populate_list_from_raw_array
+        if self.ctitem.is_long():
+            res = []
+            buf = rffi.cast(rffi.LONGP, cdata._cdata)
+            length = cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        return None
+
+    def aslist_float(self, cdata):
+        from rpython.rlib.rarray import populate_list_from_raw_array
+        if self.ctitem.is_double():
+            res = []
+            buf = rffi.cast(rffi.DOUBLEP, cdata._cdata)
+            length = cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        return None
+
     def get_vararg_type(self):
         return self.ctptr
 
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -43,13 +43,10 @@
     def is_unichar_ptr_or_array(self):
         return False
 
-    def unpack_list_of_int_items(self, cdata):
-        return None
+    def is_long(self):
+        return False
 
-    def unpack_list_of_float_items(self, cdata):
-        return None
-
-    def pack_list_of_items(self, cdata, w_ob):
+    def is_double(self):
         return False
 
     def newp(self, w_init):
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -2,7 +2,6 @@
 Primitives.
 """
 
-import sys
 from pypy.interpreter.error import operationerrfmt
 
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
@@ -170,9 +169,9 @@
             sh = self.size * 8
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
-        else:
-            self.vmin = r_uint(0)
-            self.vrangemax = r_uint(-1)
+
+    def is_long(self):
+        return self.size == rffi.sizeof(lltype.Signed)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -205,35 +204,6 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_signed_data(value)
 
-    def unpack_list_of_int_items(self, w_cdata):
-        if self.size == rffi.sizeof(rffi.LONG):
-            from rpython.rlib.rarray import populate_list_from_raw_array
-            res = []
-            buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
-            length = w_cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        elif self.value_fits_long:
-            res = [0] * w_cdata.get_array_length()
-            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
-            return res
-        return None
-
-    def pack_list_of_items(self, cdata, w_ob):
-        int_list = self.space.listview_int(w_ob)
-        if int_list is not None:
-            if self.size == rffi.sizeof(rffi.LONG): # fastest path
-                from rpython.rlib.rarray import copy_list_to_raw_array
-                cdata = rffi.cast(rffi.LONGP, cdata)
-                copy_list_to_raw_array(int_list, cdata)
-            else:
-                overflowed = misc.pack_list_to_raw_array_bounds(
-                    int_list, cdata, self.size, self.vmin, self.vrangemax)
-                if overflowed != 0:
-                    self._overflow(self.space.wrap(overflowed))
-            return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
-
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
     _attrs_            = ['value_fits_long', 'value_fits_ulong', 'vrangemax']
@@ -246,8 +216,6 @@
         self.value_fits_ulong = self.size <= rffi.sizeof(lltype.Unsigned)
         if self.value_fits_long:
             self.vrangemax = self._compute_vrange_max()
-        else:
-            self.vrangemax = r_uint(sys.maxint)
 
     def _compute_vrange_max(self):
         sh = self.size * 8
@@ -287,24 +255,6 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_unsigned_data(value)
 
-    def unpack_list_of_int_items(self, w_cdata):
-        if self.value_fits_long:
-            res = [0] * w_cdata.get_array_length()
-            misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata,
-                                                     self.size)
-            return res
-        return None
-
-    def pack_list_of_items(self, cdata, w_ob):
-        int_list = self.space.listview_int(w_ob)
-        if int_list is not None:
-            overflowed = misc.pack_list_to_raw_array_bounds(
-                int_list, cdata, self.size, r_uint(0), self.vrangemax)
-            if overflowed != 0:
-                self._overflow(self.space.wrap(overflowed))
-            return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
-
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
     _attrs_ = []
@@ -326,6 +276,9 @@
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
 
+    def is_double(self):
+        return self.size == rffi.sizeof(lltype.Float)
+
     def cast(self, w_ob):
         space = self.space
         if isinstance(w_ob, cdataobj.W_CData):
@@ -365,34 +318,6 @@
         value = space.float_w(space.float(w_ob))
         misc.write_raw_float_data(cdata, value, self.size)
 
-    def unpack_list_of_float_items(self, w_cdata):
-        if self.size == rffi.sizeof(rffi.DOUBLE):
-            from rpython.rlib.rarray import populate_list_from_raw_array
-            res = []
-            buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
-            length = w_cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        elif self.size == rffi.sizeof(rffi.FLOAT):
-            res = [0.0] * w_cdata.get_array_length()
-            misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata)
-            return res
-        return None
-
-    def pack_list_of_items(self, cdata, w_ob):
-        float_list = self.space.listview_float(w_ob)
-        if float_list is not None:
-            if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
-                from rpython.rlib.rarray import copy_list_to_raw_array
-                cdata = rffi.cast(rffi.DOUBLEP, cdata)
-                copy_list_to_raw_array(float_list, cdata)
-                return True
-            elif self.size == rffi.sizeof(rffi.FLOAT):
-                misc.pack_float_list_to_raw_array(float_list, cdata,
-                                                  rffi.FLOAT, rffi.FLOATP)
-                return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
-
 
 class W_CTypePrimitiveLongDouble(W_CTypePrimitiveFloat):
     _attrs_ = []
@@ -446,15 +371,3 @@
         else:
             value = space.float_w(space.float(w_ob))
             self._to_longdouble_and_write(value, cdata)
-
-    # Cannot have unpack_list_of_float_items() here:
-    # 'list(array-of-longdouble)' returns a list of cdata objects,
-    # not a list of floats.
-
-    def pack_list_of_items(self, cdata, w_ob):
-        float_list = self.space.listview_float(w_ob)
-        if float_list is not None:
-            misc.pack_float_list_to_raw_array(float_list, cdata,
-                                             rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
-            return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -42,6 +42,12 @@
     def is_char_or_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveCharOrUniChar)
 
+    def aslist_int(self, cdata):
+        return None
+
+    def aslist_float(self, cdata):
+        return None
+
     def cast(self, w_ob):
         # cast to a pointer, to a funcptr, or to an array.
         # Note that casting to an array is an extension to the C language,
@@ -58,10 +64,24 @@
             value = rffi.cast(rffi.CCHARP, value)
         return cdataobj.W_CData(space, value, self)
 
+    def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
+        from rpython.rlib.rarray import copy_list_to_raw_array
+        int_list = self.space.listview_int(w_ob)
+        float_list = self.space.listview_float(w_ob)
+        #
+        if self.ctitem.is_long() and int_list is not None:
+            cdata = rffi.cast(rffi.LONGP, cdata)
+            copy_list_to_raw_array(int_list, cdata)
+            return True
+        #
+        if self.ctitem.is_double() and float_list is not None:
+            cdata = rffi.cast(rffi.DOUBLEP, cdata)
+            copy_list_to_raw_array(float_list, cdata)
+            return True
+        #
+        return False
+
     def _convert_array_from_listview(self, cdata, w_ob):
-        if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
-            return
-        #
         space = self.space
         lst_w = space.listview(w_ob)
         if self.length >= 0 and len(lst_w) > self.length:
@@ -75,6 +95,11 @@
 
     def convert_array_from_object(self, cdata, w_ob):
         space = self.space
+        if self._convert_array_from_list_strategy_maybe(cdata, w_ob):
+            # the fast path worked, we are done now
+            return
+        #
+        # continue with the slow path
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             self._convert_array_from_listview(cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -315,47 +315,3 @@
             _raw_memclear_tp(TP, TPP, dest)
             return
     raise NotImplementedError("bad clear size")
-
-# ____________________________________________________________
-
-def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
-    for TP, TPP in _prim_signed_types:
-        if size == rffi.sizeof(TP):
-            ptr = rffi.cast(TPP, target)
-            for i in range(len(int_list)):
-                x = int_list[i]
-                if r_uint(x) - vmin > vrangemax:
-                    return x      # overflow
-                ptr[i] = rffi.cast(TP, x)
-            return 0
-    raise NotImplementedError("bad integer size")
-
- at specialize.arg(2)
-def pack_float_list_to_raw_array(float_list, target, TP, TPP):
-    target = rffi.cast(TPP, target)
-    for i in range(len(float_list)):
-        x = float_list[i]
-        target[i] = rffi.cast(TP, x)
-
-def unpack_list_from_raw_array(int_list, source, size):
-    for TP, TPP in _prim_signed_types:
-        if size == rffi.sizeof(TP):
-            ptr = rffi.cast(TPP, source)
-            for i in range(len(int_list)):
-                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
-            return
-    raise NotImplementedError("bad integer size")
-
-def unpack_unsigned_list_from_raw_array(int_list, source, size):
-    for TP, TPP in _prim_unsigned_types:
-        if size == rffi.sizeof(TP):
-            ptr = rffi.cast(TPP, source)
-            for i in range(len(int_list)):
-                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
-            return
-    raise NotImplementedError("bad integer size")
-
-def unpack_cfloat_list_from_raw_array(float_list, source):
-    ptr = rffi.cast(rffi.FLOATP, source)
-    for i in range(len(float_list)):
-        float_list[i] = rffi.cast(lltype.Float, ptr[i])
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -1,19 +1,18 @@
-# side-effect: FORMAT_LONGDOUBLE must be built before the first test
+# side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule()
 from pypy.module._cffi_backend import misc
-from pypy.module._cffi_backend.ctypeobj import W_CType
-
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
 
 class AppTest_fast_path_from_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
 
     def setup_method(self, meth):
-        def forbidden(*args):
+        def forbidden(self, *args):
             assert False, 'The slow path is forbidden'
-        self._original = W_CType.pack_list_of_items.im_func
-        W_CType.pack_list_of_items = forbidden
+        self._original = W_CTypePtrOrArray._convert_array_from_listview.im_func
+        W_CTypePtrOrArray._convert_array_from_listview = forbidden
 
     def teardown_method(self, meth):
-        W_CType.pack_list_of_items = self._original
+        W_CTypePtrOrArray._convert_array_from_listview = self._original
 
     def test_fast_init_from_list(self):
         import _cffi_backend
@@ -35,101 +34,6 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
-    def test_fast_init_short_from_list(self):
-        import _cffi_backend
-        SHORT = _cffi_backend.new_primitive_type('short')
-        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
-        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, None)
-        buf = _cffi_backend.newp(SHORT_ARRAY, [1, -2, 3])
-        assert buf[0] == 1
-        assert buf[1] == -2
-        assert buf[2] == 3
-        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [40000])
-        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [-40000])
-
-    def test_fast_init_longlong_from_list(self):
-        if type(2 ** 50) is long:
-            large_int = 2 ** 30
-        else:
-            large_int = 2 ** 50
-        import _cffi_backend
-        LONGLONG = _cffi_backend.new_primitive_type('long long')
-        P_LONGLONG = _cffi_backend.new_pointer_type(LONGLONG)
-        LONGLONG_ARRAY = _cffi_backend.new_array_type(P_LONGLONG, None)
-        buf = _cffi_backend.newp(LONGLONG_ARRAY, [1, -2, 3, large_int])
-        assert buf[0] == 1
-        assert buf[1] == -2
-        assert buf[2] == 3
-        assert buf[3] == large_int
-
-    def test_fast_init_ushort_from_list(self):
-        import _cffi_backend
-        USHORT = _cffi_backend.new_primitive_type('unsigned short')
-        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
-        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, None)
-        buf = _cffi_backend.newp(USHORT_ARRAY, [1, 2, 40000])
-        assert buf[0] == 1
-        assert buf[1] == 2
-        assert buf[2] == 40000
-        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [70000])
-        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [-1])
-
-    def test_fast_init_ulong_from_list(self):
-        import sys
-        import _cffi_backend
-        ULONG = _cffi_backend.new_primitive_type('unsigned long')
-        P_ULONG = _cffi_backend.new_pointer_type(ULONG)
-        ULONG_ARRAY = _cffi_backend.new_array_type(P_ULONG, None)
-        buf = _cffi_backend.newp(ULONG_ARRAY, [1, 2, sys.maxint])
-        assert buf[0] == 1
-        assert buf[1] == 2
-        assert buf[2] == sys.maxint
-        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-1])
-        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-sys.maxint])
-
-    def test_fast_init_cfloat_from_list(self):
-        import _cffi_backend
-        FLOAT = _cffi_backend.new_primitive_type('float')
-        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
-        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, None)
-        buf = _cffi_backend.newp(FLOAT_ARRAY, [1.25, -3.5])
-        assert buf[0] == 1.25
-        assert buf[1] == -3.5
-
-    def test_fast_init_clongdouble_from_list(self):
-        import _cffi_backend
-        LONGDOUBLE = _cffi_backend.new_primitive_type('long double')
-        P_LONGDOUBLE = _cffi_backend.new_pointer_type(LONGDOUBLE)
-        LONGDOUBLE_ARRAY = _cffi_backend.new_array_type(P_LONGDOUBLE, None)
-        buf = _cffi_backend.newp(LONGDOUBLE_ARRAY, [1.25, -3.5])
-        assert float(buf[0]) == 1.25
-        assert float(buf[1]) == -3.5
-
-    def test_fast_init_bool_from_list(self):
-        import _cffi_backend
-        BOOL = _cffi_backend.new_primitive_type('_Bool')
-        P_BOOL = _cffi_backend.new_pointer_type(BOOL)
-        BOOL_ARRAY = _cffi_backend.new_array_type(P_BOOL, None)
-        buf = _cffi_backend.newp(BOOL_ARRAY, [1, 0])
-        assert buf[0] == 1
-        assert buf[1] == 0
-        assert type(buf[1]) is int
-        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [2])
-        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [-1])
-
-
-class AppTest_fast_path_bug(object):
-    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
-
-    def test_bug_not_list_or_tuple(self):
-        import _cffi_backend
-        LONG = _cffi_backend.new_primitive_type('long')
-        P_LONG = _cffi_backend.new_pointer_type(LONG)
-        LONG_ARRAY_2 = _cffi_backend.new_array_type(P_LONG, 2)
-        P_LONG_ARRAY_2 = _cffi_backend.new_pointer_type(LONG_ARRAY_2)
-        LONG_ARRAY_ARRAY = _cffi_backend.new_array_type(P_LONG_ARRAY_2, None)
-        raises(TypeError, _cffi_backend.newp, LONG_ARRAY_ARRAY, [set([4, 5])])
-
 
 class AppTest_fast_path_to_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
@@ -150,38 +54,12 @@
         self._original = original
         rarray.populate_list_from_raw_array = populate_list_from_raw_array
         #
-        original2 = misc.unpack_list_from_raw_array
-        def unpack_list_from_raw_array(*args):
-            self.count += 1
-            return original2(*args)
-        self._original2 = original2
-        misc.unpack_list_from_raw_array = unpack_list_from_raw_array
-        #
-        original3 = misc.unpack_cfloat_list_from_raw_array
-        def unpack_cfloat_list_from_raw_array(*args):
-            self.count += 1
-            return original3(*args)
-        self._original3 = original3
-        misc.unpack_cfloat_list_from_raw_array = (
-            unpack_cfloat_list_from_raw_array)
-        #
-        original4 = misc.unpack_unsigned_list_from_raw_array
-        def unpack_unsigned_list_from_raw_array(*args):
-            self.count += 1
-            return original4(*args)
-        self._original4 = original4
-        misc.unpack_unsigned_list_from_raw_array = (
-            unpack_unsigned_list_from_raw_array)
-        #
         self.w_runappdirect = self.space.wrap(self.runappdirect)
 
 
     def teardown_method(self, meth):
         from rpython.rlib import rarray
         rarray.populate_list_from_raw_array = self._original
-        misc.unpack_list_from_raw_array = self._original2
-        misc.unpack_cfloat_list_from_raw_array = self._original3
-        misc.unpack_unsigned_list_from_raw_array = self._original4
 
     def test_list_int(self):
         import _cffi_backend
@@ -206,14 +84,6 @@
         pbuf = _cffi_backend.cast(P_LONG, buf)
         raises(TypeError, "list(pbuf)")
 
-    def test_bug(self):
-        import _cffi_backend
-        LONG = _cffi_backend.new_primitive_type('long')
-        five = _cffi_backend.cast(LONG, 5)
-        raises(TypeError, list, five)
-        DOUBLE = _cffi_backend.new_primitive_type('double')
-        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
-        raises(TypeError, list, five_and_a_half)
 
     def test_list_float(self):
         import _cffi_backend
@@ -228,45 +98,3 @@
         assert lst == [1.1, 2.2, 3.3]
         if not self.runappdirect:
             assert self.get_count() == 1
-
-    def test_list_short(self):
-        import _cffi_backend
-        SHORT = _cffi_backend.new_primitive_type('short')
-        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
-        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, 3)
-        buf = _cffi_backend.newp(SHORT_ARRAY)
-        buf[0] = 1
-        buf[1] = 2
-        buf[2] = 3
-        lst = list(buf)
-        assert lst == [1, 2, 3]
-        if not self.runappdirect:
-            assert self.get_count() == 1
-
-    def test_list_ushort(self):
-        import _cffi_backend
-        USHORT = _cffi_backend.new_primitive_type('unsigned short')
-        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
-        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, 3)
-        buf = _cffi_backend.newp(USHORT_ARRAY)
-        buf[0] = 1
-        buf[1] = 2
-        buf[2] = 50505
-        lst = list(buf)
-        assert lst == [1, 2, 50505]
-        if not self.runappdirect:
-            assert self.get_count() == 1
-
-    def test_list_cfloat(self):
-        import _cffi_backend
-        FLOAT = _cffi_backend.new_primitive_type('float')
-        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
-        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, 3)
-        buf = _cffi_backend.newp(FLOAT_ARRAY)
-        buf[0] = 1.25
-        buf[1] = -2.5
-        buf[2] = 3.75
-        lst = list(buf)
-        assert lst == [1.25, -2.5, 3.75]
-        if not self.runappdirect:
-            assert self.get_count() == 1
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -140,7 +140,7 @@
         ltime = rctime.localtime()
         assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
         try:
-            rctime.asctime((12345,) + (0,) * 8)  # assert this doesn't crash
+            assert rctime.asctime((12345,) + (0,) * 8).split()[-1] == '12345'
         except ValueError:
             pass  # some OS (ie POSIXes besides Linux) reject year > 9999
 
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -11,7 +11,6 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
-from rpython.translator.backendopt.graphanalyze import DependencyTracker
 
 
 class CallControl(object):
@@ -36,7 +35,6 @@
         #
         for index, jd in enumerate(jitdrivers_sd):
             jd.index = index
-        self.seen = DependencyTracker(self.readwrite_analyzer)
 
     def find_all_graphs(self, policy):
         try:
@@ -233,8 +231,8 @@
                 extraeffect = EffectInfo.EF_CANNOT_RAISE
         #
         effectinfo = effectinfo_from_writeanalyze(
-            self.readwrite_analyzer.analyze(op, self.seen), self.cpu,
-            extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
+            self.readwrite_analyzer.analyze(op), self.cpu, extraeffect,
+            oopspecindex, can_invalidate, call_release_gil_target,
         )
         #
         assert effectinfo is not None
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -82,7 +82,6 @@
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
         elif (opnum != rop.GETFIELD_GC and
-              opnum != rop.GETFIELD_GC_PURE and
               opnum != rop.MARK_OPAQUE_PTR and
               opnum != rop.PTR_EQ and
               opnum != rop.PTR_NE and
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3360,28 +3360,21 @@
         self.check_resops(call=0, getfield_gc=0)
 
     def test_isvirtual_call_assembler(self):
-        driver = JitDriver(greens = ['code'], reds = ['n', 's'])
+        driver = JitDriver(greens = ['code'], reds = ['n'])
 
         @look_inside_iff(lambda t1, t2: isvirtual(t1))
         def g(t1, t2):
             return t1[0] == t2[0]
 
-        def create(n):
-            return (1, 2, n)
-        create._dont_inline_ = True
-
         def f(code, n):
-            s = 0
             while n > 0:
-                driver.can_enter_jit(code=code, n=n, s=s)
-                driver.jit_merge_point(code=code, n=n, s=s)
-                t = create(n)
+                driver.can_enter_jit(code=code, n=n)
+                driver.jit_merge_point(code=code, n=n)
+                t = (1, 2, n)
                 if code:
                     f(0, 3)
-                s += t[2]
                 g(t, (1, 2, n))
                 n -= 1
-            return s
 
         self.meta_interp(f, [1, 10], inline=True)
         self.check_resops(call=0, call_may_force=0, call_assembler=2)
diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py
--- a/rpython/rlib/rstack.py
+++ b/rpython/rlib/rstack.py
@@ -67,7 +67,6 @@
     # Else call the slow path
     stack_check_slowpath(current)
 stack_check._always_inline_ = True
-stack_check._dont_insert_stackcheck_ = True
 
 @rgc.no_collect
 def stack_check_slowpath(current):
@@ -75,4 +74,3 @@
         from rpython.rlib.rstackovf import _StackOverflow
         raise _StackOverflow
 stack_check_slowpath._dont_inline_ = True
-stack_check_slowpath._dont_insert_stackcheck_ = True
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -97,16 +97,8 @@
     def __eq__(self, other):
         if isinstance(other, Typedef):
             return other.__eq__(self)
-        if self.__class__ is other.__class__:
-            if self is other:
-                return True
-            try:
-                if hash(self) != hash(other):
-                    return False
-            except TypeError:
-                pass # too bad, we can't use a fastpath here
-            return safe_equal(self.__dict__, other.__dict__)
-        return False
+        return self.__class__ is other.__class__ and (
+            self is other or safe_equal(self.__dict__, other.__dict__))
 
     def __ne__(self, other):
         return not (self == other)
@@ -235,9 +227,6 @@
         self.OF = OF
         self.c_name = c_name
 
-    def __hash__(self):
-        return hash(self.OF)
-
     def __repr__(self):
         return '<Typedef "%s" of %r>' % (self.c_name, self.OF)
 
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1078,13 +1078,6 @@
         res = self.interpret(func, [42])
         assert res == 42
 
-    def test_dict_with_empty_tuple_key(self):
-        def func(i):
-            d = {(): i}
-            return d[()]
-        res = self.interpret(func, [42])
-        assert res == 42
-
 
 class TestStress:
 
diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -4,7 +4,6 @@
 top_set = object()
 empty_set = frozenset()
 
-CUTOFF = 1000
 
 class WriteAnalyzer(graphanalyze.GraphAnalyzer):
     def bottom_result(self):
@@ -22,8 +21,6 @@
     def add_to_result(self, result, other):
         if other is top_set:
             return top_set
-        if len(other) + len(result) > CUTOFF:
-            return top_set
         result.update(other)
         return result
 
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -213,10 +213,6 @@
     insert_in = set()
     block2graph = {}
     for caller in translator.graphs:
-        pyobj = getattr(caller, 'func', None)
-        if pyobj is not None:
-            if getattr(pyobj, '_dont_insert_stackcheck_', False):
-                continue
         for block, callee in find_calls_from(translator, caller):
             if getattr(getattr(callee, 'func', None),
                        'insert_stack_check_here', False):
@@ -273,4 +269,4 @@
     transform_dead_op_vars(ann, block_subset)
     if ann.translator:
         checkgraphs(ann, block_subset)
-
+ 

From noreply at buildbot.pypy.org  Thu Oct 24 20:14:15 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 24 Oct 2013 20:14:15 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: copy fixes from default
Message-ID: <20131024181415.8462F1C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67583:3c7fb1b6977b
Date: 2013-10-24 11:13 -0700
http://bitbucket.org/pypy/pypy/changeset/3c7fb1b6977b/

Log:	copy fixes from default

diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -125,8 +125,7 @@
         if len(cellvars) != ncellvars:
             raise OperationError(self.space.w_TypeError,
                                  self.space.wrap("bad cellvars"))
-        if self.cells is not None:
-            self.cells[:ncellvars] = cellvars
+        self.cells[:ncellvars] = cellvars
 
     @jit.look_inside_iff(lambda self: jit.isvirtual(self))
     def fast2locals(self):
@@ -165,8 +164,6 @@
 
     @jit.unroll_safe
     def init_cells(self):
-        if not self.cells:
-            return
         args_to_copy = self.pycode._args_as_cellvars
         for i in range(len(args_to_copy)):
             argnum = args_to_copy[i]

From noreply at buildbot.pypy.org  Thu Oct 24 22:58:28 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 24 Oct 2013 22:58:28 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: Added a test
Message-ID: <20131024205828.9CB011C135D@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67584:849ecbc3d133
Date: 2013-10-24 13:57 -0700
http://bitbucket.org/pypy/pypy/changeset/849ecbc3d133/

Log:	Added a test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_min_max.py b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
--- a/pypy/module/pypyjit/test_pypy_c/test_min_max.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
@@ -1,8 +1,7 @@
-import py
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
+
 class TestMinMax(BaseTestPyPyC):
-
     def test_min_max(self):
         def main():
             i=0
@@ -24,7 +23,6 @@
             --TICK--
             jump(..., descr=...)
         """)
-        
 
     def test_silly_max(self):
         def main():
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -408,3 +408,20 @@
         log = self.run(main, [300])
         loop, = log.loops_by_id("long_op")
         assert len(loop.ops_by_id("long_op")) == 0
+
+    def test_settrace(self):
+        def main(n):
+            import sys
+            sys.settrace(lambda *args, **kwargs: None)
+
+            def f():
+                return 1
+
+            while n:
+                n -= f()
+
+        log = self.run(main, [300])
+        loops = log.loops_by_filename(self.filepath)
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(loops) == 1

From noreply at buildbot.pypy.org  Thu Oct 24 23:05:16 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 24 Oct 2013 23:05:16 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: Consistency
Message-ID: <20131024210516.B4B9F1C135D@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67585:381c742cb3e8
Date: 2013-10-24 14:04 -0700
http://bitbucket.org/pypy/pypy/changeset/381c742cb3e8/

Log:	Consistency

diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -93,7 +93,7 @@
         if self.w_locals is None:
             self.w_locals = self.space.newdict()
         varnames = self.getcode().getvarnames()
-        for i in range(min(len(varnames), self.pycode.co_nlocals)):
+        for i in range(min(len(varnames), self.getcode().co_nlocals)):
             name = varnames[i]
             w_value = self.locals_stack_w[i]
             w_name = self.space.wrap(name)
@@ -111,7 +111,7 @@
         # Copy values from self.w_locals to the fastlocals
         assert self.w_locals is not None
         varnames = self.getcode().getvarnames()
-        numlocals = self.pycode.co_nlocals
+        numlocals = self.getcode().co_nlocals
 
         new_fastlocals_w = [None] * numlocals
 

From noreply at buildbot.pypy.org  Thu Oct 24 23:14:47 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 24 Oct 2013 23:14:47 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: Fixed tests
Message-ID: <20131024211447.46F621C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67586:bab17124e8a3
Date: 2013-10-24 14:14 -0700
http://bitbucket.org/pypy/pypy/changeset/bab17124e8a3/

Log:	Fixed tests

diff --git a/pypy/interpreter/test/test_eval.py b/pypy/interpreter/test/test_eval.py
--- a/pypy/interpreter/test/test_eval.py
+++ b/pypy/interpreter/test/test_eval.py
@@ -3,67 +3,63 @@
 from pypy.interpreter.pycode import PyCode
 
 
-class TestFrame: 
+class TestFrame:
     def setup_method(self, method):
         def c(x, y, *args):
             pass
         code = PyCode._from_code(self.space, c.func_code)
 
         class ConcreteFastscopeFrame(Frame):
-            
             def __init__(self, space, code, numlocals):
                 self.code = code
+                self.code.co_nlocals = numlocals
                 Frame.__init__(self, space)
-                self.numlocals = numlocals
-                self._fastlocals_w = [None] * self.numlocals
+                self.locals_stack_w = [None] * numlocals
 
             def getcode(self):
                 return self.code
 
             def setfastscope(self, scope_w):
-                self._fastlocals_w = scope_w
+                self.locals_stack_w = scope_w
 
             def getfastscope(self):
-                return self._fastlocals_w
-
-            def getfastscopelength(self):
-                return self.numlocals
+                return self.locals_stack_w
 
         self.f = ConcreteFastscopeFrame(self.space, code, numlocals=5)
-        
+
 
     def test_fast2locals(self):
-        space = self.space 
+        space = self.space
         w = space.wrap
         self.f.fast2locals()
         assert space.eq_w(self.f.w_locals, self.space.wrap({}))
-        
-        self.f._fastlocals_w[0] = w(5)
+
+        self.f.locals_stack_w[0] = w(5)
         self.f.fast2locals()
         assert space.eq_w(self.f.w_locals, self.space.wrap({'x': 5}))
 
-        self.f._fastlocals_w[2] = w(7)
+        self.f.locals_stack_w[2] = w(7)
         self.f.fast2locals()
         assert space.eq_w(self.f.w_locals, self.space.wrap({'x': 5, 'args': 7}))
 
     def sameList(self, l1, l2):
-        assert len(l1) == len(l2) 
+        assert len(l1) == len(l2)
         for w_1, w_2 in zip(l1, l2):
             assert (w_1 is None) == (w_2 is None)
             if w_1 is not None:
-                assert self.space.eq_w(w_1, w_2) 
+                assert self.space.eq_w(w_1, w_2)
 
     def test_locals2fast(self):
         w = self.space.wrap
         self.f.w_locals = self.space.wrap({})
         self.f.locals2fast()
-        self.sameList(self.f._fastlocals_w, [None]*5)
+        self.sameList(self.f.locals_stack_w, [None] * 5)
 
         self.f.w_locals = self.space.wrap({'x': 5})
         self.f.locals2fast()
-        self.sameList(self.f._fastlocals_w, [w(5)] + [None]*4)
+        self.sameList(self.f.locals_stack_w, [w(5)] + [None] * 4)
 
-        self.f.w_locals = self.space.wrap({'x':5, 'args':7})
+        self.f.w_locals = self.space.wrap({'x': 5, 'args': 7})
         self.f.locals2fast()
-        self.sameList(self.f._fastlocals_w, [w(5), None, w(7),
+        self.sameList(self.f.locals_stack_w, [w(5), None, w(7),
                                              None, None])

From noreply at buildbot.pypy.org  Thu Oct 24 23:35:18 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 24 Oct 2013 23:35:18 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: This is quasi-immutable
Message-ID: <20131024213518.865131C31B2@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67587:a05179d6070b
Date: 2013-10-24 14:34 -0700
http://bitbucket.org/pypy/pypy/changeset/a05179d6070b/

Log:	This is quasi-immutable

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -310,6 +310,9 @@
     the GIL.  And whether we have threads or not, it is forced to zero
     whenever we fire any of the asynchronous actions.
     """
+
+    _immutable_fields_ = ["checkinterval_scaled?"]
+
     def __init__(self):
         self._periodic_actions = []
         self._nonperiodic_actions = []
diff --git a/pypy/interpreter/test/test_eval.py b/pypy/interpreter/test/test_eval.py
--- a/pypy/interpreter/test/test_eval.py
+++ b/pypy/interpreter/test/test_eval.py
@@ -27,7 +27,6 @@
 
         self.f = ConcreteFastscopeFrame(self.space, code, numlocals=5)
 
-
     def test_fast2locals(self):
         space = self.space
         w = space.wrap

From noreply at buildbot.pypy.org  Fri Oct 25 00:42:30 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 00:42:30 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: backed out my previous backout
Message-ID: <20131024224231.0316E1C0204@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67589:558bcd112473
Date: 2013-10-24 15:41 -0700
http://bitbucket.org/pypy/pypy/changeset/558bcd112473/

Log:	backed out my previous backout

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -283,10 +283,18 @@
         return self.ctype.iter(self)
 
     def unpackiterable_int(self, space):
-        return self.ctype.aslist_int(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_int_items(self)
+        return None
 
     def unpackiterable_float(self, space):
-        return self.ctype.aslist_float(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_float_items(self)
+        return None
 
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -105,26 +105,6 @@
     def iter(self, cdata):
         return W_CDataIter(self.space, self.ctitem, cdata)
 
-    def aslist_int(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_long():
-            res = []
-            buf = rffi.cast(rffi.LONGP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
-    def aslist_float(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_double():
-            res = []
-            buf = rffi.cast(rffi.DOUBLEP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
     def get_vararg_type(self):
         return self.ctptr
 
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -43,10 +43,13 @@
     def is_unichar_ptr_or_array(self):
         return False
 
-    def is_long(self):
-        return False
+    def unpack_list_of_int_items(self, cdata):
+        return None
 
-    def is_double(self):
+    def unpack_list_of_float_items(self, cdata):
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
         return False
 
     def newp(self, w_init):
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -2,6 +2,7 @@
 Primitives.
 """
 
+import sys
 from pypy.interpreter.error import operationerrfmt
 
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
@@ -169,9 +170,9 @@
             sh = self.size * 8
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
-
-    def is_long(self):
-        return self.size == rffi.sizeof(lltype.Signed)
+        else:
+            self.vmin = r_uint(0)
+            self.vrangemax = r_uint(-1)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -204,6 +205,35 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_signed_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.LONG):
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            if self.size == rffi.sizeof(rffi.LONG): # fastest path
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.LONGP, cdata)
+                copy_list_to_raw_array(int_list, cdata)
+            else:
+                overflowed = misc.pack_list_to_raw_array_bounds(
+                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                if overflowed != 0:
+                    self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
     _attrs_            = ['value_fits_long', 'value_fits_ulong', 'vrangemax']
@@ -216,6 +246,8 @@
         self.value_fits_ulong = self.size <= rffi.sizeof(lltype.Unsigned)
         if self.value_fits_long:
             self.vrangemax = self._compute_vrange_max()
+        else:
+            self.vrangemax = r_uint(sys.maxint)
 
     def _compute_vrange_max(self):
         sh = self.size * 8
@@ -255,6 +287,24 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_unsigned_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata,
+                                                     self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            overflowed = misc.pack_list_to_raw_array_bounds(
+                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            if overflowed != 0:
+                self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
     _attrs_ = []
@@ -276,9 +326,6 @@
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
 
-    def is_double(self):
-        return self.size == rffi.sizeof(lltype.Float)
-
     def cast(self, w_ob):
         space = self.space
         if isinstance(w_ob, cdataobj.W_CData):
@@ -318,6 +365,34 @@
         value = space.float_w(space.float(w_ob))
         misc.write_raw_float_data(cdata, value, self.size)
 
+    def unpack_list_of_float_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.DOUBLE):
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.size == rffi.sizeof(rffi.FLOAT):
+            res = [0.0] * w_cdata.get_array_length()
+            misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.DOUBLEP, cdata)
+                copy_list_to_raw_array(float_list, cdata)
+                return True
+            elif self.size == rffi.sizeof(rffi.FLOAT):
+                misc.pack_float_list_to_raw_array(float_list, cdata,
+                                                  rffi.FLOAT, rffi.FLOATP)
+                return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveLongDouble(W_CTypePrimitiveFloat):
     _attrs_ = []
@@ -371,3 +446,15 @@
         else:
             value = space.float_w(space.float(w_ob))
             self._to_longdouble_and_write(value, cdata)
+
+    # Cannot have unpack_list_of_float_items() here:
+    # 'list(array-of-longdouble)' returns a list of cdata objects,
+    # not a list of floats.
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            misc.pack_float_list_to_raw_array(float_list, cdata,
+                                             rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -42,12 +42,6 @@
     def is_char_or_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveCharOrUniChar)
 
-    def aslist_int(self, cdata):
-        return None
-
-    def aslist_float(self, cdata):
-        return None
-
     def cast(self, w_ob):
         # cast to a pointer, to a funcptr, or to an array.
         # Note that casting to an array is an extension to the C language,
@@ -64,24 +58,10 @@
             value = rffi.cast(rffi.CCHARP, value)
         return cdataobj.W_CData(space, value, self)
 
-    def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
-        from rpython.rlib.rarray import copy_list_to_raw_array
-        int_list = self.space.listview_int(w_ob)
-        float_list = self.space.listview_float(w_ob)
+    def _convert_array_from_listview(self, cdata, w_ob):
+        if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
+            return
         #
-        if self.ctitem.is_long() and int_list is not None:
-            cdata = rffi.cast(rffi.LONGP, cdata)
-            copy_list_to_raw_array(int_list, cdata)
-            return True
-        #
-        if self.ctitem.is_double() and float_list is not None:
-            cdata = rffi.cast(rffi.DOUBLEP, cdata)
-            copy_list_to_raw_array(float_list, cdata)
-            return True
-        #
-        return False
-
-    def _convert_array_from_listview(self, cdata, w_ob):
         space = self.space
         lst_w = space.listview(w_ob)
         if self.length >= 0 and len(lst_w) > self.length:
@@ -95,11 +75,6 @@
 
     def convert_array_from_object(self, cdata, w_ob):
         space = self.space
-        if self._convert_array_from_list_strategy_maybe(cdata, w_ob):
-            # the fast path worked, we are done now
-            return
-        #
-        # continue with the slow path
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             self._convert_array_from_listview(cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -315,3 +315,47 @@
             _raw_memclear_tp(TP, TPP, dest)
             return
     raise NotImplementedError("bad clear size")
+
+# ____________________________________________________________
+
+def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) - vmin > vrangemax:
+                    return x      # overflow
+                ptr[i] = rffi.cast(TP, x)
+            return 0
+    raise NotImplementedError("bad integer size")
+
+ at specialize.arg(2)
+def pack_float_list_to_raw_array(float_list, target, TP, TPP):
+    target = rffi.cast(TPP, target)
+    for i in range(len(float_list)):
+        x = float_list[i]
+        target[i] = rffi.cast(TP, x)
+
+def unpack_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_unsigned_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_unsigned_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_cfloat_list_from_raw_array(float_list, source):
+    ptr = rffi.cast(rffi.FLOATP, source)
+    for i in range(len(float_list)):
+        float_list[i] = rffi.cast(lltype.Float, ptr[i])
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -1,18 +1,19 @@
-# side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule()
+# side-effect: FORMAT_LONGDOUBLE must be built before the first test
 from pypy.module._cffi_backend import misc
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeobj import W_CType
+
 
 class AppTest_fast_path_from_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
 
     def setup_method(self, meth):
-        def forbidden(self, *args):
+        def forbidden(*args):
             assert False, 'The slow path is forbidden'
-        self._original = W_CTypePtrOrArray._convert_array_from_listview.im_func
-        W_CTypePtrOrArray._convert_array_from_listview = forbidden
+        self._original = W_CType.pack_list_of_items.im_func
+        W_CType.pack_list_of_items = forbidden
 
     def teardown_method(self, meth):
-        W_CTypePtrOrArray._convert_array_from_listview = self._original
+        W_CType.pack_list_of_items = self._original
 
     def test_fast_init_from_list(self):
         import _cffi_backend
@@ -34,6 +35,101 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
+    def test_fast_init_short_from_list(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, None)
+        buf = _cffi_backend.newp(SHORT_ARRAY, [1, -2, 3])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [40000])
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [-40000])
+
+    def test_fast_init_longlong_from_list(self):
+        if type(2 ** 50) is long:
+            large_int = 2 ** 30
+        else:
+            large_int = 2 ** 50
+        import _cffi_backend
+        LONGLONG = _cffi_backend.new_primitive_type('long long')
+        P_LONGLONG = _cffi_backend.new_pointer_type(LONGLONG)
+        LONGLONG_ARRAY = _cffi_backend.new_array_type(P_LONGLONG, None)
+        buf = _cffi_backend.newp(LONGLONG_ARRAY, [1, -2, 3, large_int])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        assert buf[3] == large_int
+
+    def test_fast_init_ushort_from_list(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, None)
+        buf = _cffi_backend.newp(USHORT_ARRAY, [1, 2, 40000])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == 40000
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [70000])
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [-1])
+
+    def test_fast_init_ulong_from_list(self):
+        import sys
+        import _cffi_backend
+        ULONG = _cffi_backend.new_primitive_type('unsigned long')
+        P_ULONG = _cffi_backend.new_pointer_type(ULONG)
+        ULONG_ARRAY = _cffi_backend.new_array_type(P_ULONG, None)
+        buf = _cffi_backend.newp(ULONG_ARRAY, [1, 2, sys.maxint])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == sys.maxint
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-1])
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-sys.maxint])
+
+    def test_fast_init_cfloat_from_list(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, None)
+        buf = _cffi_backend.newp(FLOAT_ARRAY, [1.25, -3.5])
+        assert buf[0] == 1.25
+        assert buf[1] == -3.5
+
+    def test_fast_init_clongdouble_from_list(self):
+        import _cffi_backend
+        LONGDOUBLE = _cffi_backend.new_primitive_type('long double')
+        P_LONGDOUBLE = _cffi_backend.new_pointer_type(LONGDOUBLE)
+        LONGDOUBLE_ARRAY = _cffi_backend.new_array_type(P_LONGDOUBLE, None)
+        buf = _cffi_backend.newp(LONGDOUBLE_ARRAY, [1.25, -3.5])
+        assert float(buf[0]) == 1.25
+        assert float(buf[1]) == -3.5
+
+    def test_fast_init_bool_from_list(self):
+        import _cffi_backend
+        BOOL = _cffi_backend.new_primitive_type('_Bool')
+        P_BOOL = _cffi_backend.new_pointer_type(BOOL)
+        BOOL_ARRAY = _cffi_backend.new_array_type(P_BOOL, None)
+        buf = _cffi_backend.newp(BOOL_ARRAY, [1, 0])
+        assert buf[0] == 1
+        assert buf[1] == 0
+        assert type(buf[1]) is int
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [2])
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [-1])
+
+
+class AppTest_fast_path_bug(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def test_bug_not_list_or_tuple(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY_2 = _cffi_backend.new_array_type(P_LONG, 2)
+        P_LONG_ARRAY_2 = _cffi_backend.new_pointer_type(LONG_ARRAY_2)
+        LONG_ARRAY_ARRAY = _cffi_backend.new_array_type(P_LONG_ARRAY_2, None)
+        raises(TypeError, _cffi_backend.newp, LONG_ARRAY_ARRAY, [set([4, 5])])
+
 
 class AppTest_fast_path_to_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
@@ -54,12 +150,38 @@
         self._original = original
         rarray.populate_list_from_raw_array = populate_list_from_raw_array
         #
+        original2 = misc.unpack_list_from_raw_array
+        def unpack_list_from_raw_array(*args):
+            self.count += 1
+            return original2(*args)
+        self._original2 = original2
+        misc.unpack_list_from_raw_array = unpack_list_from_raw_array
+        #
+        original3 = misc.unpack_cfloat_list_from_raw_array
+        def unpack_cfloat_list_from_raw_array(*args):
+            self.count += 1
+            return original3(*args)
+        self._original3 = original3
+        misc.unpack_cfloat_list_from_raw_array = (
+            unpack_cfloat_list_from_raw_array)
+        #
+        original4 = misc.unpack_unsigned_list_from_raw_array
+        def unpack_unsigned_list_from_raw_array(*args):
+            self.count += 1
+            return original4(*args)
+        self._original4 = original4
+        misc.unpack_unsigned_list_from_raw_array = (
+            unpack_unsigned_list_from_raw_array)
+        #
         self.w_runappdirect = self.space.wrap(self.runappdirect)
 
 
     def teardown_method(self, meth):
         from rpython.rlib import rarray
         rarray.populate_list_from_raw_array = self._original
+        misc.unpack_list_from_raw_array = self._original2
+        misc.unpack_cfloat_list_from_raw_array = self._original3
+        misc.unpack_unsigned_list_from_raw_array = self._original4
 
     def test_list_int(self):
         import _cffi_backend
@@ -84,6 +206,14 @@
         pbuf = _cffi_backend.cast(P_LONG, buf)
         raises(TypeError, "list(pbuf)")
 
+    def test_bug(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        five = _cffi_backend.cast(LONG, 5)
+        raises(TypeError, list, five)
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
+        raises(TypeError, list, five_and_a_half)
 
     def test_list_float(self):
         import _cffi_backend
@@ -98,3 +228,45 @@
         assert lst == [1.1, 2.2, 3.3]
         if not self.runappdirect:
             assert self.get_count() == 1
+
+    def test_list_short(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, 3)
+        buf = _cffi_backend.newp(SHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_ushort(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, 3)
+        buf = _cffi_backend.newp(USHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 50505
+        lst = list(buf)
+        assert lst == [1, 2, 50505]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_cfloat(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, 3)
+        buf = _cffi_backend.newp(FLOAT_ARRAY)
+        buf[0] = 1.25
+        buf[1] = -2.5
+        buf[2] = 3.75
+        lst = list(buf)
+        assert lst == [1.25, -2.5, 3.75]
+        if not self.runappdirect:
+            assert self.get_count() == 1
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -140,7 +140,7 @@
         ltime = rctime.localtime()
         assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
         try:
-            assert rctime.asctime((12345,) + (0,) * 8).split()[-1] == '12345'
+            rctime.asctime((12345,) + (0,) * 8)  # assert this doesn't crash
         except ValueError:
             pass  # some OS (ie POSIXes besides Linux) reject year > 9999
 
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -11,6 +11,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
+from rpython.translator.backendopt.graphanalyze import DependencyTracker
 
 
 class CallControl(object):
@@ -38,6 +39,7 @@
         #
         for index, jd in enumerate(jitdrivers_sd):
             jd.index = index
+        self.seen = DependencyTracker(self.readwrite_analyzer)
 
     def find_all_graphs(self, policy):
         try:
@@ -234,8 +236,8 @@
                 extraeffect = EffectInfo.EF_CANNOT_RAISE
         #
         effectinfo = effectinfo_from_writeanalyze(
-            self.readwrite_analyzer.analyze(op), self.cpu, extraeffect,
-            oopspecindex, can_invalidate, call_release_gil_target,
+            self.readwrite_analyzer.analyze(op, self.seen), self.cpu,
+            extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
         )
         #
         assert effectinfo is not None
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -82,6 +82,7 @@
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
         elif (opnum != rop.GETFIELD_GC and
+              opnum != rop.GETFIELD_GC_PURE and
               opnum != rop.MARK_OPAQUE_PTR and
               opnum != rop.PTR_EQ and
               opnum != rop.PTR_NE and
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3360,21 +3360,28 @@
         self.check_resops(call=0, getfield_gc=0)
 
     def test_isvirtual_call_assembler(self):
-        driver = JitDriver(greens = ['code'], reds = ['n'])
+        driver = JitDriver(greens = ['code'], reds = ['n', 's'])
 
         @look_inside_iff(lambda t1, t2: isvirtual(t1))
         def g(t1, t2):
             return t1[0] == t2[0]
 
+        def create(n):
+            return (1, 2, n)
+        create._dont_inline_ = True
+
         def f(code, n):
+            s = 0
             while n > 0:
-                driver.can_enter_jit(code=code, n=n)
-                driver.jit_merge_point(code=code, n=n)
-                t = (1, 2, n)
+                driver.can_enter_jit(code=code, n=n, s=s)
+                driver.jit_merge_point(code=code, n=n, s=s)
+                t = create(n)
                 if code:
                     f(0, 3)
+                s += t[2]
                 g(t, (1, 2, n))
                 n -= 1
+            return s
 
         self.meta_interp(f, [1, 10], inline=True)
         self.check_resops(call=0, call_may_force=0, call_assembler=2)
diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py
--- a/rpython/rlib/rstack.py
+++ b/rpython/rlib/rstack.py
@@ -67,6 +67,7 @@
     # Else call the slow path
     stack_check_slowpath(current)
 stack_check._always_inline_ = True
+stack_check._dont_insert_stackcheck_ = True
 
 @rgc.no_collect
 def stack_check_slowpath(current):
@@ -74,3 +75,4 @@
         from rpython.rlib.rstackovf import _StackOverflow
         raise _StackOverflow
 stack_check_slowpath._dont_inline_ = True
+stack_check_slowpath._dont_insert_stackcheck_ = True
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -97,8 +97,16 @@
     def __eq__(self, other):
         if isinstance(other, Typedef):
             return other.__eq__(self)
-        return self.__class__ is other.__class__ and (
-            self is other or safe_equal(self.__dict__, other.__dict__))
+        if self.__class__ is other.__class__:
+            if self is other:
+                return True
+            try:
+                if hash(self) != hash(other):
+                    return False
+            except TypeError:
+                pass # too bad, we can't use a fastpath here
+            return safe_equal(self.__dict__, other.__dict__)
+        return False
 
     def __ne__(self, other):
         return not (self == other)
@@ -227,6 +235,9 @@
         self.OF = OF
         self.c_name = c_name
 
+    def __hash__(self):
+        return hash(self.OF)
+
     def __repr__(self):
         return '<Typedef "%s" of %r>' % (self.c_name, self.OF)
 
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1078,6 +1078,13 @@
         res = self.interpret(func, [42])
         assert res == 42
 
+    def test_dict_with_empty_tuple_key(self):
+        def func(i):
+            d = {(): i}
+            return d[()]
+        res = self.interpret(func, [42])
+        assert res == 42
+
 
 class TestStress:
 
diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -4,6 +4,7 @@
 top_set = object()
 empty_set = frozenset()
 
+CUTOFF = 1000
 
 class WriteAnalyzer(graphanalyze.GraphAnalyzer):
     def bottom_result(self):
@@ -21,6 +22,8 @@
     def add_to_result(self, result, other):
         if other is top_set:
             return top_set
+        if len(other) + len(result) > CUTOFF:
+            return top_set
         result.update(other)
         return result
 
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -213,6 +213,10 @@
     insert_in = set()
     block2graph = {}
     for caller in translator.graphs:
+        pyobj = getattr(caller, 'func', None)
+        if pyobj is not None:
+            if getattr(pyobj, '_dont_insert_stackcheck_', False):
+                continue
         for block, callee in find_calls_from(translator, caller):
             if getattr(getattr(callee, 'func', None),
                        'insert_stack_check_here', False):
@@ -269,4 +273,4 @@
     transform_dead_op_vars(ann, block_subset)
     if ann.translator:
         checkgraphs(ann, block_subset)
- 
+

From noreply at buildbot.pypy.org  Fri Oct 25 04:49:56 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 25 Oct 2013 04:49:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: attempt to fix the logic introduced in
	72c1e31
Message-ID: <20131025024956.072511C00D8@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67590:3e3897b5cbd0
Date: 2013-10-24 22:48 -0400
http://bitbucket.org/pypy/pypy/changeset/3e3897b5cbd0/

Log:	attempt to fix the logic introduced in 72c1e31

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -101,8 +101,8 @@
             if self is other:
                 return True
             try:
-                if hash(self) != hash(other):
-                    return False
+                if hash(self) == hash(other):
+                    return True
             except TypeError:
                 pass # too bad, we can't use a fastpath here
             return safe_equal(self.__dict__, other.__dict__)

From noreply at buildbot.pypy.org  Fri Oct 25 04:55:38 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 25 Oct 2013 04:55:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: Backed out changeset 3e3897b5cbd0
Message-ID: <20131025025538.875151C00D8@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67591:e7dec9b071db
Date: 2013-10-24 22:54 -0400
http://bitbucket.org/pypy/pypy/changeset/e7dec9b071db/

Log:	Backed out changeset 3e3897b5cbd0

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -101,8 +101,8 @@
             if self is other:
                 return True
             try:
-                if hash(self) == hash(other):
-                    return True
+                if hash(self) != hash(other):
+                    return False
             except TypeError:
                 pass # too bad, we can't use a fastpath here
             return safe_equal(self.__dict__, other.__dict__)

From noreply at buildbot.pypy.org  Fri Oct 25 05:02:29 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 25 Oct 2013 05:02:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: back out 72c1e31475e9 and d1ee8abf66b0
	to fix breakage
Message-ID: <20131025030229.B8D6C1C135D@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67592:cd2a4dd3979a
Date: 2013-10-24 22:59 -0400
http://bitbucket.org/pypy/pypy/changeset/cd2a4dd3979a/

Log:	back out 72c1e31475e9 and d1ee8abf66b0 to fix breakage

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -97,16 +97,8 @@
     def __eq__(self, other):
         if isinstance(other, Typedef):
             return other.__eq__(self)
-        if self.__class__ is other.__class__:
-            if self is other:
-                return True
-            try:
-                if hash(self) != hash(other):
-                    return False
-            except TypeError:
-                pass # too bad, we can't use a fastpath here
-            return safe_equal(self.__dict__, other.__dict__)
-        return False
+        return self.__class__ is other.__class__ and (
+            self is other or safe_equal(self.__dict__, other.__dict__))
 
     def __ne__(self, other):
         return not (self == other)
@@ -235,9 +227,6 @@
         self.OF = OF
         self.c_name = c_name
 
-    def __hash__(self):
-        return hash(self.OF)
-
     def __repr__(self):
         return '<Typedef "%s" of %r>' % (self.c_name, self.OF)
 

From noreply at buildbot.pypy.org  Fri Oct 25 05:02:31 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 25 Oct 2013 05:02:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131025030231.98CF21C135D@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67593:db7a73ea7e32
Date: 2013-10-24 22:59 -0400
http://bitbucket.org/pypy/pypy/changeset/db7a73ea7e32/

Log:	merge

diff --git a/lib_pypy/numpypy/lib/utils.py b/lib_pypy/numpypy/lib/utils.py
--- a/lib_pypy/numpypy/lib/utils.py
+++ b/lib_pypy/numpypy/lib/utils.py
@@ -21,14 +21,4 @@
         ...
 
     """
-    try:
-        import numpy
-    except:
-        # running from pypy source directory
-        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-        return os.path.join(head, '../include')
-    else:
-        # using installed numpy core headers
-        import numpy.core as core
-        d = os.path.join(os.path.dirname(core.__file__), 'include')
-    return d
+    return os.path.join(os.path.dirname(__file__), '../../../include')
diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -125,8 +125,7 @@
         if len(cellvars) != ncellvars:
             raise OperationError(self.space.w_TypeError,
                                  self.space.wrap("bad cellvars"))
-        if self.cells is not None:
-            self.cells[:ncellvars] = cellvars
+        self.cells[:ncellvars] = cellvars
 
     @jit.dont_look_inside
     def fast2locals(self):
@@ -165,8 +164,6 @@
 
     @jit.unroll_safe
     def init_cells(self):
-        if self.cells is None:
-            return
         args_to_copy = self.pycode._args_as_cellvars
         for i in range(len(args_to_copy)):
             argnum = args_to_copy[i]
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -283,10 +283,18 @@
         return self.ctype.iter(self)
 
     def unpackiterable_int(self, space):
-        return self.ctype.aslist_int(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_int_items(self)
+        return None
 
     def unpackiterable_float(self, space):
-        return self.ctype.aslist_float(self)
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_float_items(self)
+        return None
 
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -105,26 +105,6 @@
     def iter(self, cdata):
         return W_CDataIter(self.space, self.ctitem, cdata)
 
-    def aslist_int(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_long():
-            res = []
-            buf = rffi.cast(rffi.LONGP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
-    def aslist_float(self, cdata):
-        from rpython.rlib.rarray import populate_list_from_raw_array
-        if self.ctitem.is_double():
-            res = []
-            buf = rffi.cast(rffi.DOUBLEP, cdata._cdata)
-            length = cdata.get_array_length()
-            populate_list_from_raw_array(res, buf, length)
-            return res
-        return None
-
     def get_vararg_type(self):
         return self.ctptr
 
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -43,10 +43,13 @@
     def is_unichar_ptr_or_array(self):
         return False
 
-    def is_long(self):
-        return False
+    def unpack_list_of_int_items(self, cdata):
+        return None
 
-    def is_double(self):
+    def unpack_list_of_float_items(self, cdata):
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
         return False
 
     def newp(self, w_init):
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -2,6 +2,7 @@
 Primitives.
 """
 
+import sys
 from pypy.interpreter.error import operationerrfmt
 
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
@@ -169,9 +170,9 @@
             sh = self.size * 8
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
-
-    def is_long(self):
-        return self.size == rffi.sizeof(lltype.Signed)
+        else:
+            self.vmin = r_uint(0)
+            self.vrangemax = r_uint(-1)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -204,6 +205,35 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_signed_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.LONG):
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            if self.size == rffi.sizeof(rffi.LONG): # fastest path
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.LONGP, cdata)
+                copy_list_to_raw_array(int_list, cdata)
+            else:
+                overflowed = misc.pack_list_to_raw_array_bounds(
+                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                if overflowed != 0:
+                    self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
     _attrs_            = ['value_fits_long', 'value_fits_ulong', 'vrangemax']
@@ -216,6 +246,8 @@
         self.value_fits_ulong = self.size <= rffi.sizeof(lltype.Unsigned)
         if self.value_fits_long:
             self.vrangemax = self._compute_vrange_max()
+        else:
+            self.vrangemax = r_uint(sys.maxint)
 
     def _compute_vrange_max(self):
         sh = self.size * 8
@@ -255,6 +287,24 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_unsigned_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata,
+                                                     self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            overflowed = misc.pack_list_to_raw_array_bounds(
+                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            if overflowed != 0:
+                self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
     _attrs_ = []
@@ -276,9 +326,6 @@
 class W_CTypePrimitiveFloat(W_CTypePrimitive):
     _attrs_ = []
 
-    def is_double(self):
-        return self.size == rffi.sizeof(lltype.Float)
-
     def cast(self, w_ob):
         space = self.space
         if isinstance(w_ob, cdataobj.W_CData):
@@ -318,6 +365,34 @@
         value = space.float_w(space.float(w_ob))
         misc.write_raw_float_data(cdata, value, self.size)
 
+    def unpack_list_of_float_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.DOUBLE):
+            from rpython.rlib.rarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.size == rffi.sizeof(rffi.FLOAT):
+            res = [0.0] * w_cdata.get_array_length()
+            misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
+                from rpython.rlib.rarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.DOUBLEP, cdata)
+                copy_list_to_raw_array(float_list, cdata)
+                return True
+            elif self.size == rffi.sizeof(rffi.FLOAT):
+                misc.pack_float_list_to_raw_array(float_list, cdata,
+                                                  rffi.FLOAT, rffi.FLOATP)
+                return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveLongDouble(W_CTypePrimitiveFloat):
     _attrs_ = []
@@ -328,7 +403,6 @@
         return misc.longdouble2str(lvalue)
 
     def cast(self, w_ob):
-        space = self.space
         if (isinstance(w_ob, cdataobj.W_CData) and
                 isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)):
             w_cdata = self.convert_to_object(w_ob._cdata)
@@ -372,3 +446,15 @@
         else:
             value = space.float_w(space.float(w_ob))
             self._to_longdouble_and_write(value, cdata)
+
+    # Cannot have unpack_list_of_float_items() here:
+    # 'list(array-of-longdouble)' returns a list of cdata objects,
+    # not a list of floats.
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            misc.pack_float_list_to_raw_array(float_list, cdata,
+                                             rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -42,12 +42,6 @@
     def is_char_or_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveCharOrUniChar)
 
-    def aslist_int(self, cdata):
-        return None
-
-    def aslist_float(self, cdata):
-        return None
-
     def cast(self, w_ob):
         # cast to a pointer, to a funcptr, or to an array.
         # Note that casting to an array is an extension to the C language,
@@ -64,24 +58,10 @@
             value = rffi.cast(rffi.CCHARP, value)
         return cdataobj.W_CData(space, value, self)
 
-    def _convert_array_from_list_strategy_maybe(self, cdata, w_ob):
-        from rpython.rlib.rarray import copy_list_to_raw_array
-        int_list = self.space.listview_int(w_ob)
-        float_list = self.space.listview_float(w_ob)
+    def _convert_array_from_listview(self, cdata, w_ob):
+        if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
+            return
         #
-        if self.ctitem.is_long() and int_list is not None:
-            cdata = rffi.cast(rffi.LONGP, cdata)
-            copy_list_to_raw_array(int_list, cdata)
-            return True
-        #
-        if self.ctitem.is_double() and float_list is not None:
-            cdata = rffi.cast(rffi.DOUBLEP, cdata)
-            copy_list_to_raw_array(float_list, cdata)
-            return True
-        #
-        return False
-
-    def _convert_array_from_listview(self, cdata, w_ob):
         space = self.space
         lst_w = space.listview(w_ob)
         if self.length >= 0 and len(lst_w) > self.length:
@@ -95,11 +75,6 @@
 
     def convert_array_from_object(self, cdata, w_ob):
         space = self.space
-        if self._convert_array_from_list_strategy_maybe(cdata, w_ob):
-            # the fast path worked, we are done now
-            return
-        #
-        # continue with the slow path
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             self._convert_array_from_listview(cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -315,3 +315,47 @@
             _raw_memclear_tp(TP, TPP, dest)
             return
     raise NotImplementedError("bad clear size")
+
+# ____________________________________________________________
+
+def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) - vmin > vrangemax:
+                    return x      # overflow
+                ptr[i] = rffi.cast(TP, x)
+            return 0
+    raise NotImplementedError("bad integer size")
+
+ at specialize.arg(2)
+def pack_float_list_to_raw_array(float_list, target, TP, TPP):
+    target = rffi.cast(TPP, target)
+    for i in range(len(float_list)):
+        x = float_list[i]
+        target[i] = rffi.cast(TP, x)
+
+def unpack_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_unsigned_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_unsigned_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_cfloat_list_from_raw_array(float_list, source):
+    ptr = rffi.cast(rffi.FLOATP, source)
+    for i in range(len(float_list)):
+        float_list[i] = rffi.cast(lltype.Float, ptr[i])
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -117,13 +117,17 @@
 
 SF_MSVC_BITFIELDS = 1
 SF_GCC_ARM_BITFIELDS = 2
+SF_GCC_BIG_ENDIAN = 4
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
-elif rffi_platform.getdefined('__arm__', ''):
-    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
-    DEFAULT_SFLAGS = 0
+    if rffi_platform.getdefined('__arm__', ''):
+        DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
+    else:
+        DEFAULT_SFLAGS = 0
+    if sys.byteorder == 'big':
+        DEFAULT_SFLAGS |= SF_GCC_BIG_ENDIAN
 
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
@@ -325,6 +329,9 @@
                     prev_bitfield_free -= fbitsize
                     field_offset_bytes = boffset / 8 - ftype.size
 
+                if sflags & SF_GCC_BIG_ENDIAN:
+                    bitshift = 8 * ftype.size - fbitsize- bitshift
+
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
                                            bitshift, fbitsize)
                 fields_list.append(fld)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2949,8 +2949,6 @@
     _test_bitfield_details(flag=2)
 
 def test_bitfield_as_big_endian():
-    if '__pypy__' in sys.builtin_module_names:
-        py.test.skip("no big endian machine supported on pypy for now")
     _test_bitfield_details(flag=4)
 
 
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -1,18 +1,19 @@
-# side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule()
+# side-effect: FORMAT_LONGDOUBLE must be built before the first test
 from pypy.module._cffi_backend import misc
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeobj import W_CType
+
 
 class AppTest_fast_path_from_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
 
     def setup_method(self, meth):
-        def forbidden(self, *args):
+        def forbidden(*args):
             assert False, 'The slow path is forbidden'
-        self._original = W_CTypePtrOrArray._convert_array_from_listview.im_func
-        W_CTypePtrOrArray._convert_array_from_listview = forbidden
+        self._original = W_CType.pack_list_of_items.im_func
+        W_CType.pack_list_of_items = forbidden
 
     def teardown_method(self, meth):
-        W_CTypePtrOrArray._convert_array_from_listview = self._original
+        W_CType.pack_list_of_items = self._original
 
     def test_fast_init_from_list(self):
         import _cffi_backend
@@ -34,6 +35,101 @@
         assert buf[1] == 2.2
         assert buf[2] == 3.3
 
+    def test_fast_init_short_from_list(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, None)
+        buf = _cffi_backend.newp(SHORT_ARRAY, [1, -2, 3])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [40000])
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [-40000])
+
+    def test_fast_init_longlong_from_list(self):
+        if type(2 ** 50) is long:
+            large_int = 2 ** 30
+        else:
+            large_int = 2 ** 50
+        import _cffi_backend
+        LONGLONG = _cffi_backend.new_primitive_type('long long')
+        P_LONGLONG = _cffi_backend.new_pointer_type(LONGLONG)
+        LONGLONG_ARRAY = _cffi_backend.new_array_type(P_LONGLONG, None)
+        buf = _cffi_backend.newp(LONGLONG_ARRAY, [1, -2, 3, large_int])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        assert buf[3] == large_int
+
+    def test_fast_init_ushort_from_list(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, None)
+        buf = _cffi_backend.newp(USHORT_ARRAY, [1, 2, 40000])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == 40000
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [70000])
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [-1])
+
+    def test_fast_init_ulong_from_list(self):
+        import sys
+        import _cffi_backend
+        ULONG = _cffi_backend.new_primitive_type('unsigned long')
+        P_ULONG = _cffi_backend.new_pointer_type(ULONG)
+        ULONG_ARRAY = _cffi_backend.new_array_type(P_ULONG, None)
+        buf = _cffi_backend.newp(ULONG_ARRAY, [1, 2, sys.maxint])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == sys.maxint
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-1])
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-sys.maxint])
+
+    def test_fast_init_cfloat_from_list(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, None)
+        buf = _cffi_backend.newp(FLOAT_ARRAY, [1.25, -3.5])
+        assert buf[0] == 1.25
+        assert buf[1] == -3.5
+
+    def test_fast_init_clongdouble_from_list(self):
+        import _cffi_backend
+        LONGDOUBLE = _cffi_backend.new_primitive_type('long double')
+        P_LONGDOUBLE = _cffi_backend.new_pointer_type(LONGDOUBLE)
+        LONGDOUBLE_ARRAY = _cffi_backend.new_array_type(P_LONGDOUBLE, None)
+        buf = _cffi_backend.newp(LONGDOUBLE_ARRAY, [1.25, -3.5])
+        assert float(buf[0]) == 1.25
+        assert float(buf[1]) == -3.5
+
+    def test_fast_init_bool_from_list(self):
+        import _cffi_backend
+        BOOL = _cffi_backend.new_primitive_type('_Bool')
+        P_BOOL = _cffi_backend.new_pointer_type(BOOL)
+        BOOL_ARRAY = _cffi_backend.new_array_type(P_BOOL, None)
+        buf = _cffi_backend.newp(BOOL_ARRAY, [1, 0])
+        assert buf[0] == 1
+        assert buf[1] == 0
+        assert type(buf[1]) is int
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [2])
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [-1])
+
+
+class AppTest_fast_path_bug(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def test_bug_not_list_or_tuple(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY_2 = _cffi_backend.new_array_type(P_LONG, 2)
+        P_LONG_ARRAY_2 = _cffi_backend.new_pointer_type(LONG_ARRAY_2)
+        LONG_ARRAY_ARRAY = _cffi_backend.new_array_type(P_LONG_ARRAY_2, None)
+        raises(TypeError, _cffi_backend.newp, LONG_ARRAY_ARRAY, [set([4, 5])])
+
 
 class AppTest_fast_path_to_list(object):
     spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
@@ -54,12 +150,38 @@
         self._original = original
         rarray.populate_list_from_raw_array = populate_list_from_raw_array
         #
+        original2 = misc.unpack_list_from_raw_array
+        def unpack_list_from_raw_array(*args):
+            self.count += 1
+            return original2(*args)
+        self._original2 = original2
+        misc.unpack_list_from_raw_array = unpack_list_from_raw_array
+        #
+        original3 = misc.unpack_cfloat_list_from_raw_array
+        def unpack_cfloat_list_from_raw_array(*args):
+            self.count += 1
+            return original3(*args)
+        self._original3 = original3
+        misc.unpack_cfloat_list_from_raw_array = (
+            unpack_cfloat_list_from_raw_array)
+        #
+        original4 = misc.unpack_unsigned_list_from_raw_array
+        def unpack_unsigned_list_from_raw_array(*args):
+            self.count += 1
+            return original4(*args)
+        self._original4 = original4
+        misc.unpack_unsigned_list_from_raw_array = (
+            unpack_unsigned_list_from_raw_array)
+        #
         self.w_runappdirect = self.space.wrap(self.runappdirect)
 
 
     def teardown_method(self, meth):
         from rpython.rlib import rarray
         rarray.populate_list_from_raw_array = self._original
+        misc.unpack_list_from_raw_array = self._original2
+        misc.unpack_cfloat_list_from_raw_array = self._original3
+        misc.unpack_unsigned_list_from_raw_array = self._original4
 
     def test_list_int(self):
         import _cffi_backend
@@ -84,6 +206,14 @@
         pbuf = _cffi_backend.cast(P_LONG, buf)
         raises(TypeError, "list(pbuf)")
 
+    def test_bug(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        five = _cffi_backend.cast(LONG, 5)
+        raises(TypeError, list, five)
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
+        raises(TypeError, list, five_and_a_half)
 
     def test_list_float(self):
         import _cffi_backend
@@ -98,3 +228,45 @@
         assert lst == [1.1, 2.2, 3.3]
         if not self.runappdirect:
             assert self.get_count() == 1
+
+    def test_list_short(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, 3)
+        buf = _cffi_backend.newp(SHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_ushort(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, 3)
+        buf = _cffi_backend.newp(USHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 50505
+        lst = list(buf)
+        assert lst == [1, 2, 50505]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_cfloat(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, 3)
+        buf = _cffi_backend.newp(FLOAT_ARRAY)
+        buf[0] = 1.25
+        buf[1] = -2.5
+        buf[2] = 3.75
+        lst = list(buf)
+        assert lst == [1.25, -2.5, 3.75]
+        if not self.runappdirect:
+            assert self.get_count() == 1
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -84,6 +84,7 @@
 
     def build_and_convert(self, space, box):
         return self.itemtype.build_and_convert(space, self, box)
+
     def coerce(self, space, w_item):
         return self.itemtype.coerce(space, self, w_item)
 
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -140,7 +140,7 @@
         ltime = rctime.localtime()
         assert rctime.asctime(tuple(ltime)) == rctime.asctime(ltime)
         try:
-            assert rctime.asctime((12345,) + (0,) * 8).split()[-1] == '12345'
+            rctime.asctime((12345,) + (0,) * 8)  # assert this doesn't crash
         except ValueError:
             pass  # some OS (ie POSIXes besides Linux) reject year > 9999
 
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -11,6 +11,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
+from rpython.translator.backendopt.graphanalyze import DependencyTracker
 
 
 class CallControl(object):
@@ -32,6 +33,9 @@
             self.virtualizable_analyzer = VirtualizableAnalyzer(translator)
             self.quasiimmut_analyzer = QuasiImmutAnalyzer(translator)
             self.randomeffects_analyzer = RandomEffectsAnalyzer(translator)
+            self.seen = DependencyTracker(self.readwrite_analyzer)
+        else:
+            self.seen = None
         #
         for index, jd in enumerate(jitdrivers_sd):
             jd.index = index
@@ -231,8 +235,8 @@
                 extraeffect = EffectInfo.EF_CANNOT_RAISE
         #
         effectinfo = effectinfo_from_writeanalyze(
-            self.readwrite_analyzer.analyze(op), self.cpu, extraeffect,
-            oopspecindex, can_invalidate, call_release_gil_target,
+            self.readwrite_analyzer.analyze(op, self.seen), self.cpu,
+            extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
         )
         #
         assert effectinfo is not None
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -93,6 +93,8 @@
         block.exitswitch = renamings.get(block.exitswitch, block.exitswitch)
         self.follow_constant_exit(block)
         self.optimize_goto_if_not(block)
+        if isinstance(block.exitswitch, tuple):
+            self._check_no_vable_array(block.exitswitch)
         for link in block.exits:
             self._check_no_vable_array(link.args)
             self._do_renaming_on_link(renamings, link)
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -84,6 +84,8 @@
     def calldescr_canraise(self, calldescr):
         return calldescr is not self._descr_cannot_raise and calldescr.oopspecindex == EffectInfo.OS_NONE
     def get_vinfo(self, VTYPEPTR):
+        if hasattr(VTYPEPTR.TO, 'inst_vlist'):
+            return FakeVInfo()
         return None
 
 class FakeCallControlWithVRefInfo:
@@ -100,6 +102,13 @@
     def calldescr_canraise(self, calldescr):
         return False
 
+class FakeVInfo:
+    static_field_to_extra_box = {}
+    array_fields = {'inst_vlist': '?'}
+    array_field_counter = {'inst_vlist': 0}
+    array_field_descrs = [FakeDescr()]
+    array_descrs = [FakeDescr()]
+
 # ____________________________________________________________
 
 def test_reorder_renaming_list():
@@ -1001,6 +1010,22 @@
             float_return %(result_var)s
         """ % {"result_var": result_var, "tmp_var": tmp_var}, transform=True)
 
+    def test_vable_attribute_list_is_not_None(self):
+        class F:
+            _virtualizable_ = ['vlist[*]']
+            vlist = None
+            def __init__(self, x):
+                self.vlist = [x]
+        def g():
+            return F(42)
+        def f():
+            f = g()
+            if f.vlist is not None:
+                pass
+        e = py.test.raises(AssertionError, self.encoding_test, f, [], "!",
+                           transform=True)
+        assert str(e.value).startswith("A virtualizable array is passed aroun")
+
 
 def check_force_cast(FROM, TO, operations, value):
     """Check that the test is correctly written..."""
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -82,6 +82,7 @@
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
         elif (opnum != rop.GETFIELD_GC and
+              opnum != rop.GETFIELD_GC_PURE and
               opnum != rop.MARK_OPAQUE_PTR and
               opnum != rop.PTR_EQ and
               opnum != rop.PTR_NE and
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1398,7 +1398,7 @@
                     assembler_call_jd)
             if resbox is not None:
                 self.make_result_of_lastop(resbox)
-            self.metainterp.vable_after_residual_call()
+            self.metainterp.vable_after_residual_call(funcbox)
             self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None)
             if vablebox is not None:
                 self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None)
@@ -2437,7 +2437,7 @@
                 # it by ConstPtr(NULL).
                 self.stop_tracking_virtualref(i)
 
-    def vable_after_residual_call(self):
+    def vable_after_residual_call(self, funcbox):
         vinfo = self.jitdriver_sd.virtualizable_info
         if vinfo is not None:
             virtualizable_box = self.virtualizable_boxes[-1]
@@ -2445,6 +2445,14 @@
             if vinfo.tracing_after_residual_call(virtualizable):
                 # the virtualizable escaped during CALL_MAY_FORCE.
                 self.load_fields_from_virtualizable()
+                target_name = self.staticdata.get_name_from_address(funcbox.getaddr())
+                if target_name:
+                    target_name = "ConstClass(%s)" % target_name
+                else:
+                    target_name = str(funcbox.getaddr())
+                debug_print('vable escaped during a call in %s to %s' % (
+                    self.framestack[-1].jitcode.name, target_name
+                ))
                 raise SwitchToBlackhole(Counters.ABORT_ESCAPE,
                                         raising_exception=True)
                 # ^^^ we set 'raising_exception' to True because we must still
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3360,21 +3360,28 @@
         self.check_resops(call=0, getfield_gc=0)
 
     def test_isvirtual_call_assembler(self):
-        driver = JitDriver(greens = ['code'], reds = ['n'])
+        driver = JitDriver(greens = ['code'], reds = ['n', 's'])
 
         @look_inside_iff(lambda t1, t2: isvirtual(t1))
         def g(t1, t2):
             return t1[0] == t2[0]
 
+        def create(n):
+            return (1, 2, n)
+        create._dont_inline_ = True
+
         def f(code, n):
+            s = 0
             while n > 0:
-                driver.can_enter_jit(code=code, n=n)
-                driver.jit_merge_point(code=code, n=n)
-                t = (1, 2, n)
+                driver.can_enter_jit(code=code, n=n, s=s)
+                driver.jit_merge_point(code=code, n=n, s=s)
+                t = create(n)
                 if code:
                     f(0, 3)
+                s += t[2]
                 g(t, (1, 2, n))
                 n -= 1
+            return s
 
         self.meta_interp(f, [1, 10], inline=True)
         self.check_resops(call=0, call_may_force=0, call_assembler=2)
diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py
--- a/rpython/rlib/rstack.py
+++ b/rpython/rlib/rstack.py
@@ -67,6 +67,7 @@
     # Else call the slow path
     stack_check_slowpath(current)
 stack_check._always_inline_ = True
+stack_check._dont_insert_stackcheck_ = True
 
 @rgc.no_collect
 def stack_check_slowpath(current):
@@ -74,3 +75,4 @@
         from rpython.rlib.rstackovf import _StackOverflow
         raise _StackOverflow
 stack_check_slowpath._dont_inline_ = True
+stack_check_slowpath._dont_insert_stackcheck_ = True
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1078,6 +1078,13 @@
         res = self.interpret(func, [42])
         assert res == 42
 
+    def test_dict_with_empty_tuple_key(self):
+        def func(i):
+            d = {(): i}
+            return d[()]
+        res = self.interpret(func, [42])
+        assert res == 42
+
 
 class TestStress:
 
diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -4,6 +4,7 @@
 top_set = object()
 empty_set = frozenset()
 
+CUTOFF = 1000
 
 class WriteAnalyzer(graphanalyze.GraphAnalyzer):
     def bottom_result(self):
@@ -21,6 +22,8 @@
     def add_to_result(self, result, other):
         if other is top_set:
             return top_set
+        if len(other) + len(result) > CUTOFF:
+            return top_set
         result.update(other)
         return result
 
diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py
--- a/rpython/translator/transform.py
+++ b/rpython/translator/transform.py
@@ -213,6 +213,10 @@
     insert_in = set()
     block2graph = {}
     for caller in translator.graphs:
+        pyobj = getattr(caller, 'func', None)
+        if pyobj is not None:
+            if getattr(pyobj, '_dont_insert_stackcheck_', False):
+                continue
         for block, callee in find_calls_from(translator, caller):
             if getattr(getattr(callee, 'func', None),
                        'insert_stack_check_here', False):
@@ -269,4 +273,4 @@
     transform_dead_op_vars(ann, block_subset)
     if ann.translator:
         checkgraphs(ann, block_subset)
- 
+

From noreply at buildbot.pypy.org  Fri Oct 25 05:04:13 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 05:04:13 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: mergedd efault
Message-ID: <20131025030413.CFDC21C135D@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67594:664414f1f20e
Date: 2013-10-24 20:03 -0700
http://bitbucket.org/pypy/pypy/changeset/664414f1f20e/

Log:	mergedd efault

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -97,16 +97,8 @@
     def __eq__(self, other):
         if isinstance(other, Typedef):
             return other.__eq__(self)
-        if self.__class__ is other.__class__:
-            if self is other:
-                return True
-            try:
-                if hash(self) != hash(other):
-                    return False
-            except TypeError:
-                pass # too bad, we can't use a fastpath here
-            return safe_equal(self.__dict__, other.__dict__)
-        return False
+        return self.__class__ is other.__class__ and (
+            self is other or safe_equal(self.__dict__, other.__dict__))
 
     def __ne__(self, other):
         return not (self == other)
@@ -235,9 +227,6 @@
         self.OF = OF
         self.c_name = c_name
 
-    def __hash__(self):
-        return hash(self.OF)
-
     def __repr__(self):
         return '<Typedef "%s" of %r>' % (self.c_name, self.OF)
 

From noreply at buildbot.pypy.org  Fri Oct 25 11:19:04 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 25 Oct 2013 11:19:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Mention rpython.rlib.*
Message-ID: <20131025091904.7AB1C1C042B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67595:1e3a1707f812
Date: 2013-10-25 11:18 +0200
http://bitbucket.org/pypy/pypy/changeset/1e3a1707f812/

Log:	Mention rpython.rlib.*

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -266,6 +266,7 @@
 You cannot use most existing standard library modules from RPython.  The
 exceptions are
 some functions in ``os``, ``math`` and ``time`` that have native support.
+We have our own "RPython standard library" in ``rpython.rlib.*``.
 
 To read more about the RPython limitations read the `RPython description`_.
 

From noreply at buildbot.pypy.org  Fri Oct 25 11:24:27 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 25 Oct 2013 11:24:27 +0200 (CEST)
Subject: [pypy-commit] pypy default: Rename rlib/rarray to rlib/rrawarray to
 avoid confusion along
Message-ID: <20131025092427.913E11C042B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67596:1f1e196acd0d
Date: 2013-10-25 11:23 +0200
http://bitbucket.org/pypy/pypy/changeset/1f1e196acd0d/

Log:	Rename rlib/rarray to rlib/rrawarray to avoid confusion along the
	lines of "it's an RPython version of the array module!".

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -207,7 +207,7 @@
 
     def unpack_list_of_int_items(self, w_cdata):
         if self.size == rffi.sizeof(rffi.LONG):
-            from rpython.rlib.rarray import populate_list_from_raw_array
+            from rpython.rlib.rrawarray import populate_list_from_raw_array
             res = []
             buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
             length = w_cdata.get_array_length()
@@ -223,7 +223,7 @@
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
             if self.size == rffi.sizeof(rffi.LONG): # fastest path
-                from rpython.rlib.rarray import copy_list_to_raw_array
+                from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
@@ -367,7 +367,7 @@
 
     def unpack_list_of_float_items(self, w_cdata):
         if self.size == rffi.sizeof(rffi.DOUBLE):
-            from rpython.rlib.rarray import populate_list_from_raw_array
+            from rpython.rlib.rrawarray import populate_list_from_raw_array
             res = []
             buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
             length = w_cdata.get_array_length()
@@ -383,7 +383,7 @@
         float_list = self.space.listview_float(w_ob)
         if float_list is not None:
             if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
-                from rpython.rlib.rarray import copy_list_to_raw_array
+                from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.DOUBLEP, cdata)
                 copy_list_to_raw_array(float_list, cdata)
                 return True
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -136,19 +136,19 @@
 
     def setup_method(self, meth):
         from pypy.interpreter import gateway
-        from rpython.rlib import rarray
+        from rpython.rlib import rrawarray
         #
         self.count = 0
         def get_count(*args):
             return self.space.wrap(self.count)
         self.w_get_count = self.space.wrap(gateway.interp2app(get_count))
         #
-        original = rarray.populate_list_from_raw_array
+        original = rrawarray.populate_list_from_raw_array
         def populate_list_from_raw_array(*args):
             self.count += 1
             return original(*args)
         self._original = original
-        rarray.populate_list_from_raw_array = populate_list_from_raw_array
+        rrawarray.populate_list_from_raw_array = populate_list_from_raw_array
         #
         original2 = misc.unpack_list_from_raw_array
         def unpack_list_from_raw_array(*args):
@@ -177,8 +177,8 @@
 
 
     def teardown_method(self, meth):
-        from rpython.rlib import rarray
-        rarray.populate_list_from_raw_array = self._original
+        from rpython.rlib import rrawarray
+        rrawarray.populate_list_from_raw_array = self._original
         misc.unpack_list_from_raw_array = self._original2
         misc.unpack_cfloat_list_from_raw_array = self._original3
         misc.unpack_unsigned_list_from_raw_array = self._original4
diff --git a/rpython/rlib/rarray.py b/rpython/rlib/rrawarray.py
rename from rpython/rlib/rarray.py
rename to rpython/rlib/rrawarray.py
--- a/rpython/rlib/rarray.py
+++ b/rpython/rlib/rrawarray.py
@@ -1,5 +1,4 @@
 from rpython.annotator import model as annmodel
-from rpython.annotator.listdef import ListDef
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, llmemory
diff --git a/rpython/rlib/test/test_rarray.py b/rpython/rlib/test/test_rrawarray.py
rename from rpython/rlib/test/test_rarray.py
rename to rpython/rlib/test/test_rrawarray.py
--- a/rpython/rlib/test/test_rarray.py
+++ b/rpython/rlib/test/test_rrawarray.py
@@ -1,4 +1,5 @@
-from rpython.rlib.rarray import copy_list_to_raw_array, populate_list_from_raw_array
+from rpython.rlib.rrawarray import copy_list_to_raw_array, \
+                                   populate_list_from_raw_array
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.test.tool import BaseRtypingTest
 

From noreply at buildbot.pypy.org  Fri Oct 25 12:38:36 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 25 Oct 2013 12:38:36 +0200 (CEST)
Subject: [pypy-commit] stmgc default: make allocate_public_integer_address
 use the h_original (not always allocate stubs) if it is already public
Message-ID: <20131025103836.E906A1C12CC@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r540:ba0819e4b5e7
Date: 2013-10-25 12:38 +0200
http://bitbucket.org/pypy/stmgc/changeset/ba0819e4b5e7/

Log:	make allocate_public_integer_address use the h_original (not always
	allocate stubs) if it is already public

diff --git a/c4/demo_random.c b/c4/demo_random.c
--- a/c4/demo_random.c
+++ b/c4/demo_random.c
@@ -276,9 +276,14 @@
         intptr_t ip = td.public_ints[i];
         gcptr obj = (gcptr)ip;
         assert(obj->h_tid & GCFLAG_PUBLIC);
-        assert(obj->h_tid & GCFLAG_SMALLSTUB);
+        assert((obj->h_tid & GCFLAG_SMALLSTUB)
+               || (obj->h_original == 0 
+                   || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL));
         check(obj);
-        check((gcptr)(obj->h_revision - 2));
+        if (obj->h_revision & 2)
+            check((gcptr)(obj->h_revision - 2));
+        else if ((obj->h_revision & 3) == 0)
+            check((gcptr)(obj->h_revision));
     }
 }
 
diff --git a/c4/et.c b/c4/et.c
--- a/c4/et.c
+++ b/c4/et.c
@@ -1549,8 +1549,8 @@
                 a transaction) */
 
   /* XXX */
-  fprintf(stderr, "[%lx] inevitable: %s\n",
-           (long)d->public_descriptor_index, why);
+  /* fprintf(stderr, "[%lx] inevitable: %s\n", */
+  /*          (long)d->public_descriptor_index, why); */
   dprintf(("[%lx] inevitable: %s\n",
            (long)d->public_descriptor_index, why));
 
diff --git a/c4/extra.c b/c4/extra.c
--- a/c4/extra.c
+++ b/c4/extra.c
@@ -85,24 +85,39 @@
 
     spinlock_acquire(d->public_descriptor->collection_lock, 'P');
 
-    stub = stm_stub_malloc(d->public_descriptor, 0);
-    stub->h_tid = (obj->h_tid & STM_USER_TID_MASK)
-        | GCFLAG_PUBLIC | GCFLAG_STUB | GCFLAG_SMALLSTUB
-        | GCFLAG_OLD;
-
-    stub->h_revision = ((revision_t)obj) | 2;
-    if (!(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL) && obj->h_original) {
-        stub->h_original = obj->h_original;
+    /* it must have a h_original */
+    gcptr orig;
+    if (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL) {
+        orig = obj;
+    } else {
+        orig = (gcptr)obj->h_original;
+    }
+    
+    if (orig->h_tid & GCFLAG_PUBLIC) {
+        /* the original is public, so we can take that as a non-movable
+         object to register */
+        result = (intptr_t)orig;
     }
     else {
-        stub->h_original = (revision_t)obj;
+        stub = stm_stub_malloc(d->public_descriptor, 0);
+        stub->h_tid = (obj->h_tid & STM_USER_TID_MASK)
+            | GCFLAG_PUBLIC | GCFLAG_STUB | GCFLAG_SMALLSTUB
+            | GCFLAG_OLD;
+
+        stub->h_revision = ((revision_t)obj) | 2;
+        if (!(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL) && obj->h_original) {
+            stub->h_original = obj->h_original;
+        }
+        else {
+            stub->h_original = (revision_t)obj;
+        }
+
+        result = (intptr_t)stub;
     }
-
-    result = (intptr_t)stub;
     spinlock_release(d->public_descriptor->collection_lock);
     stm_register_integer_address(result);
 
-    dprintf(("allocate_public_int_adr(%p): %p", obj, stub));
+    dprintf(("allocate_public_int_adr(%p): %p", obj, (void*)result));
     return result;
 }
 
diff --git a/c4/gcpage.c b/c4/gcpage.c
--- a/c4/gcpage.c
+++ b/c4/gcpage.c
@@ -22,8 +22,9 @@
 /* Only computed during a major collection */
 static size_t mc_total_in_use, mc_total_reserved;
 
-/* keeps track of registered smallstubs that will survive unless unregistered */
-static struct G2L registered_stubs;
+/* keeps track of registered *public* objects that will survive
+unless unregistered. For now, only smallstubs and h_originals allowed */
+static struct G2L registered_objs;
 
 /* For tests */
 long stmgcpage_count(int quantity)
@@ -66,7 +67,7 @@
             (GC_PAGE_SIZE - sizeof(page_header_t)) / (WORD * i);
     }
 
-    memset(&registered_stubs, 0, sizeof(registered_stubs));
+    memset(&registered_objs, 0, sizeof(registered_objs));
 }
 
 void stmgcpage_init_tls(void)
@@ -218,27 +219,50 @@
 
 void stm_register_integer_address(intptr_t adr)
 {
+    wlog_t *found;
     gcptr obj = (gcptr)adr;
-    assert(obj->h_tid & GCFLAG_SMALLSTUB);
+    /* current limitations for 'adr': smallstub or h_original */
+    assert((obj->h_tid & GCFLAG_SMALLSTUB)
+           || (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL));
     assert(obj->h_tid & GCFLAG_PUBLIC);
 
     stmgcpage_acquire_global_lock();
-    g2l_insert(&registered_stubs, obj, NULL);
+
+    /* find and increment refcount; or insert */
+    G2L_FIND(registered_objs, obj, found, goto not_found);
+    found->val = (gcptr)(((revision_t)found->val) + 1);
+    goto finish;
+ not_found:
+    g2l_insert(&registered_objs, obj, (gcptr)1);
+
+ finish:
     stmgcpage_release_global_lock();
     dprintf(("registered %p\n", obj));
 }
 
 void stm_unregister_integer_address(intptr_t adr)
 {
+    wlog_t *found;
     gcptr obj = (gcptr)adr;
-    assert(obj->h_tid & GCFLAG_SMALLSTUB);
+
+    assert((obj->h_tid & GCFLAG_SMALLSTUB)
+           || (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL));
     assert(obj->h_tid & GCFLAG_PUBLIC);
 
     stmgcpage_acquire_global_lock();
-    int deleted = g2l_delete_item(&registered_stubs, obj);
-    assert(deleted);
+
+    /* find and decrement refcount */
+    G2L_FIND(registered_objs, obj, found, goto not_found);
+    found->val = (gcptr)(((revision_t)found->val) - 1);
+    if (found->val == NULL)
+        found->addr = NULL;     /* delete it */
+
     stmgcpage_release_global_lock();
     dprintf(("unregistered %p\n", obj));
+    return;
+
+ not_found:
+    assert(0);                  /* unmatched unregister */
 }
 
 
@@ -495,34 +519,40 @@
     }
 }
 
-static void mark_registered_stubs(void)
+static void mark_registered_objs(void)
 {
     wlog_t *item;
     gcptr L;
 
-    G2L_LOOP_FORWARD(registered_stubs, item) {
+    G2L_LOOP_FORWARD(registered_objs, item) {
         gcptr R = item->addr;
-        assert(R->h_tid & GCFLAG_SMALLSTUB);
-        /* The following assert can fail if we have a stub pointing to
-           a stub and both are registered_stubs.  This case is benign. */
-        //assert(!(R->h_tid & (GCFLAG_VISITED | GCFLAG_MARKED)));
+        assert(R->h_tid & GCFLAG_PUBLIC);
 
-        R->h_tid |= (GCFLAG_MARKED | GCFLAG_VISITED);
-
-        if (R->h_revision & 2) {
-            L = (gcptr)(R->h_revision - 2);
-            L = stmgcpage_visit(L);
-            R->h_revision = ((revision_t)L) | 2;
+        if ((R->h_original == 0) || (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)) {
+            /* the obj is an original and will therefore survive: */
+            gcptr V = stmgcpage_visit(R);
+            assert(V == R);
         }
         else {
-            L = (gcptr)R->h_revision;
-            L = stmgcpage_visit(L);
-            R->h_revision = (revision_t)L;
+            assert(R->h_tid & GCFLAG_SMALLSTUB); /* only case for now */
+            /* make sure R stays valid: */
+            R->h_tid |= (GCFLAG_MARKED | GCFLAG_VISITED);
+
+            if (R->h_revision & 2) {
+                L = (gcptr)(R->h_revision - 2);
+                L = stmgcpage_visit(L);
+                R->h_revision = ((revision_t)L) | 2;
+            }
+            else {
+                L = (gcptr)R->h_revision;
+                L = stmgcpage_visit(L);
+                R->h_revision = (revision_t)L;
+            }
+
+            /* h_original will be kept up-to-date because
+               it is either == L or L's h_original. And
+               h_originals don't move */
         }
-
-        /* h_original will be kept up-to-date because
-           it is either == L or L's h_original. And
-           h_originals don't move */
     } G2L_LOOP_END;
 
 }
@@ -961,7 +991,7 @@
 
     assert(gcptrlist_size(&objects_to_trace) == 0);
     mark_prebuilt_roots();
-    mark_registered_stubs();
+    mark_registered_objs();
     mark_all_stack_roots();
     
     /* weakrefs: */
diff --git a/c4/test/test_extra.py b/c4/test/test_extra.py
--- a/c4/test/test_extra.py
+++ b/c4/test/test_extra.py
@@ -230,7 +230,9 @@
 
     # we have stubs here:
     assert ffi.cast("gcptr", p1p).h_tid & GCFLAG_PUBLIC
-    assert classify(ffi.cast("gcptr", p1p)) == 'stub'
+    p1pp = ffi.cast("gcptr", p1p)
+    assert p1 == p1pp
+    assert classify(p1pp) == 'public'
     assert classify(ffi.cast("gcptr", p2p)) == 'stub'
     assert classify(ffi.cast("gcptr", p3p)) == 'stub'
 

From noreply at buildbot.pypy.org  Fri Oct 25 13:23:33 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 25 Oct 2013 13:23:33 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: support something more
 advanced in writebarrier before copy (only llimpl
Message-ID: <20131025112333.0E4411C019E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67597:7048d98f967f
Date: 2013-10-25 13:22 +0200
http://bitbucket.org/pypy/pypy/changeset/7048d98f967f/

Log:	support something more advanced in writebarrier before copy (only
	llimpl actually, since the actual one seems to work), use
	ll_arraycopy in rdict

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -7,6 +7,7 @@
 from rpython.rlib.objectmodel import we_are_translated, enforceargs, specialize
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rlib.objectmodel import keepalive_until_here
 
 # ____________________________________________________________
 # General GC features
@@ -137,15 +138,38 @@
         hop.exception_cannot_occur()
         return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype)
 
-def copy_struct_item(source, dest, si, di):
+ at specialize.ll()
+def copy_struct_item(source, dest, si, di, emit_write_barrier=True):
+    """ Copy struct items. There are two versions - one that emits
+    write barrier and one that does not. The one that does not *must*
+    have write barrier called before the copy
+    """
+    TP = lltype.typeOf(source).TO
+    if emit_write_barrier:
+        _copy_struct_item(source, dest, si, di)
+    else:
+        source_addr = llmemory.cast_ptr_to_adr(source)
+        dest_addr   = llmemory.cast_ptr_to_adr(dest)
+        cp_source_addr = (source_addr + llmemory.itemoffsetof(TP, 0) +
+                          llmemory.sizeof(TP.OF) * si)
+        cp_dest_addr = (dest_addr + llmemory.itemoffsetof(TP, 0) +
+                        llmemory.sizeof(TP.OF) * di)
+        llmemory.raw_memcopy(cp_source_addr, cp_dest_addr,
+                             llmemory.sizeof(TP.OF))
+        keepalive_until_here(source)
+        keepalive_until_here(dest)
+copy_struct_item._always_inline_ = True
+
+def _copy_struct_item(source, dest, si, di):
     TP = lltype.typeOf(source).TO.OF
     i = 0
     while i < len(TP._names):
-        setattr(dest[di], TP._names[i], getattr(source[si], TP._names[i]))
+        setattr(dest[di], TP._names[i],
+                getattr(source[si], TP._names[i]))
         i += 1
 
 class CopyStructEntry(ExtRegistryEntry):
-    _about_ = copy_struct_item
+    _about_ = _copy_struct_item
 
     def compute_result_annotation(self, s_source, s_dest, si, di):
         pass
@@ -189,7 +213,6 @@
 @specialize.ll()
 def ll_arraycopy(source, dest, source_start, dest_start, length):
     from rpython.rtyper.lltypesystem.lloperation import llop
-    from rpython.rlib.objectmodel import keepalive_until_here
 
     # XXX: Hack to ensure that we get a proper effectinfo.write_descrs_arrays
     # and also, maybe, speed up very small cases
@@ -266,7 +289,7 @@
     func._dont_inline_ = True
     func._no_release_gil_ = True
     return func
-    
+
 def no_collect(func):
     func._dont_inline_ = True
     func._gc_no_collect_ = True
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -228,3 +228,18 @@
     x1 = X()
     n = rgc.get_rpy_memory_usage(rgc.cast_instance_to_gcref(x1))
     assert n >= 8 and n <= 64
+
+def test_copy_struct_items_no_wb():
+    S = lltype.GcArray(lltype.Struct('x', ('a', lltype.Signed), ('b', lltype.Signed)))
+
+    def f():
+        a = lltype.malloc(S, 1)
+        a[0].a = 3
+        a[0].b = 13
+        b = lltype.malloc(S, 1)
+        rgc.copy_struct_item(a, b, 0, 0, False)
+        assert b[0].a == 3
+        assert b[0].b == 13
+
+    f()
+    interpret(f, [])
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -522,8 +522,10 @@
     A = lltype.typeOf(source)
     assert A == lltype.typeOf(dest)
     if isinstance(A.TO, lltype.GcArray):
-        assert isinstance(A.TO.OF, lltype.Ptr)
-        assert A.TO.OF.TO._gckind == 'gc'
+        if isinstance(A.TO.OF, lltype.Ptr):
+            assert A.TO.OF.TO._gckind == 'gc'
+        else:
+            assert isinstance(A.TO.OF, lltype.Struct)
     else:
         assert isinstance(A.TO, lltype.GcStruct)
         assert A.TO._arrayfld is not None
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -624,20 +624,7 @@
         return True
 
     newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
-    #
-    # XXX we should do this with rgc.ll_arraycopy()!!
-    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
-    i = 0
-    while i < len(d.entries):
-        src = d.entries[i]
-        dst = newitems[i]
-        dst.key = src.key
-        dst.value = src.value
-        if hasattr(ENTRY, 'f_hash'):
-            dst.f_hash = src.f_hash
-        if hasattr(ENTRY, 'f_valid'):
-            dst.f_valid = src.f_valid
-        i += 1
+    rgc.ll_arraycopy(d.entries, newitems, 0, 0, len(d.entries))
     d.entries = newitems
     return False
 

From noreply at buildbot.pypy.org  Fri Oct 25 17:37:11 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 17:37:11 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: a new program to debug
Message-ID: <20131025153711.1B80A1C0050@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67598:9ce613a941bb
Date: 2013-10-25 08:36 -0700
http://bitbucket.org/pypy/pypy/changeset/9ce613a941bb/

Log:	a new program to debug

diff --git a/pypy/tool/pypyjit_demo.py b/pypy/tool/pypyjit_demo.py
--- a/pypy/tool/pypyjit_demo.py
+++ b/pypy/tool/pypyjit_demo.py
@@ -1,22 +1,8 @@
+def f():
+    i = 0
+    while i < 1303:
+        i += 1
+    return i
 
-def g(i):
-    k = 0
-    while k < 3:
-        k += 1
-    return i + 1
 
-def f(x):
-    for i in range(10000):
-        t = (1, 2, i)
-        i = g(i)
-        x == t
-
-
-
-try:
-    f((1, 2, 3))
-
-except Exception, e:
-    print "Exception: ", type(e)
-    print e
-
+f()

From noreply at buildbot.pypy.org  Fri Oct 25 17:52:54 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 25 Oct 2013 17:52:54 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: import stmgc with better constptr
	allocation
Message-ID: <20131025155254.831951C0050@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67599:a4d501389912
Date: 2013-10-25 14:33 +0200
http://bitbucket.org/pypy/pypy/changeset/a4d501389912/

Log:	import stmgc with better constptr allocation

diff --git a/rpython/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c
--- a/rpython/translator/stm/src_stm/et.c
+++ b/rpython/translator/stm/src_stm/et.c
@@ -1550,8 +1550,8 @@
                 a transaction) */
 
   /* XXX */
-  fprintf(stderr, "[%lx] inevitable: %s\n",
-           (long)d->public_descriptor_index, why);
+  /* fprintf(stderr, "[%lx] inevitable: %s\n", */
+  /*          (long)d->public_descriptor_index, why); */
   dprintf(("[%lx] inevitable: %s\n",
            (long)d->public_descriptor_index, why));
 
diff --git a/rpython/translator/stm/src_stm/extra.c b/rpython/translator/stm/src_stm/extra.c
--- a/rpython/translator/stm/src_stm/extra.c
+++ b/rpython/translator/stm/src_stm/extra.c
@@ -86,24 +86,39 @@
 
     spinlock_acquire(d->public_descriptor->collection_lock, 'P');
 
-    stub = stm_stub_malloc(d->public_descriptor, 0);
-    stub->h_tid = (obj->h_tid & STM_USER_TID_MASK)
-        | GCFLAG_PUBLIC | GCFLAG_STUB | GCFLAG_SMALLSTUB
-        | GCFLAG_OLD;
-
-    stub->h_revision = ((revision_t)obj) | 2;
-    if (!(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL) && obj->h_original) {
-        stub->h_original = obj->h_original;
+    /* it must have a h_original */
+    gcptr orig;
+    if (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL) {
+        orig = obj;
+    } else {
+        orig = (gcptr)obj->h_original;
+    }
+    
+    if (orig->h_tid & GCFLAG_PUBLIC) {
+        /* the original is public, so we can take that as a non-movable
+         object to register */
+        result = (intptr_t)orig;
     }
     else {
-        stub->h_original = (revision_t)obj;
+        stub = stm_stub_malloc(d->public_descriptor, 0);
+        stub->h_tid = (obj->h_tid & STM_USER_TID_MASK)
+            | GCFLAG_PUBLIC | GCFLAG_STUB | GCFLAG_SMALLSTUB
+            | GCFLAG_OLD;
+
+        stub->h_revision = ((revision_t)obj) | 2;
+        if (!(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL) && obj->h_original) {
+            stub->h_original = obj->h_original;
+        }
+        else {
+            stub->h_original = (revision_t)obj;
+        }
+
+        result = (intptr_t)stub;
     }
-
-    result = (intptr_t)stub;
     spinlock_release(d->public_descriptor->collection_lock);
     stm_register_integer_address(result);
 
-    dprintf(("allocate_public_int_adr(%p): %p", obj, stub));
+    dprintf(("allocate_public_int_adr(%p): %p", obj, (void*)result));
     return result;
 }
 
diff --git a/rpython/translator/stm/src_stm/gcpage.c b/rpython/translator/stm/src_stm/gcpage.c
--- a/rpython/translator/stm/src_stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/gcpage.c
@@ -23,8 +23,9 @@
 /* Only computed during a major collection */
 static size_t mc_total_in_use, mc_total_reserved;
 
-/* keeps track of registered smallstubs that will survive unless unregistered */
-static struct G2L registered_stubs;
+/* keeps track of registered *public* objects that will survive
+unless unregistered. For now, only smallstubs and h_originals allowed */
+static struct G2L registered_objs;
 
 /* For tests */
 long stmgcpage_count(int quantity)
@@ -67,7 +68,7 @@
             (GC_PAGE_SIZE - sizeof(page_header_t)) / (WORD * i);
     }
 
-    memset(&registered_stubs, 0, sizeof(registered_stubs));
+    memset(&registered_objs, 0, sizeof(registered_objs));
 }
 
 void stmgcpage_init_tls(void)
@@ -219,27 +220,50 @@
 
 void stm_register_integer_address(intptr_t adr)
 {
+    wlog_t *found;
     gcptr obj = (gcptr)adr;
-    assert(obj->h_tid & GCFLAG_SMALLSTUB);
+    /* current limitations for 'adr': smallstub or h_original */
+    assert((obj->h_tid & GCFLAG_SMALLSTUB)
+           || (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL));
     assert(obj->h_tid & GCFLAG_PUBLIC);
 
     stmgcpage_acquire_global_lock();
-    g2l_insert(&registered_stubs, obj, NULL);
+
+    /* find and increment refcount; or insert */
+    G2L_FIND(registered_objs, obj, found, goto not_found);
+    found->val = (gcptr)(((revision_t)found->val) + 1);
+    goto finish;
+ not_found:
+    g2l_insert(&registered_objs, obj, (gcptr)1);
+
+ finish:
     stmgcpage_release_global_lock();
     dprintf(("registered %p\n", obj));
 }
 
 void stm_unregister_integer_address(intptr_t adr)
 {
+    wlog_t *found;
     gcptr obj = (gcptr)adr;
-    assert(obj->h_tid & GCFLAG_SMALLSTUB);
+
+    assert((obj->h_tid & GCFLAG_SMALLSTUB)
+           || (obj->h_original == 0 || obj->h_tid & GCFLAG_PREBUILT_ORIGINAL));
     assert(obj->h_tid & GCFLAG_PUBLIC);
 
     stmgcpage_acquire_global_lock();
-    int deleted = g2l_delete_item(&registered_stubs, obj);
-    assert(deleted);
+
+    /* find and decrement refcount */
+    G2L_FIND(registered_objs, obj, found, goto not_found);
+    found->val = (gcptr)(((revision_t)found->val) - 1);
+    if (found->val == NULL)
+        found->addr = NULL;     /* delete it */
+
     stmgcpage_release_global_lock();
     dprintf(("unregistered %p\n", obj));
+    return;
+
+ not_found:
+    assert(0);                  /* unmatched unregister */
 }
 
 
@@ -496,34 +520,40 @@
     }
 }
 
-static void mark_registered_stubs(void)
+static void mark_registered_objs(void)
 {
     wlog_t *item;
     gcptr L;
 
-    G2L_LOOP_FORWARD(registered_stubs, item) {
+    G2L_LOOP_FORWARD(registered_objs, item) {
         gcptr R = item->addr;
-        assert(R->h_tid & GCFLAG_SMALLSTUB);
-        /* The following assert can fail if we have a stub pointing to
-           a stub and both are registered_stubs.  This case is benign. */
-        //assert(!(R->h_tid & (GCFLAG_VISITED | GCFLAG_MARKED)));
+        assert(R->h_tid & GCFLAG_PUBLIC);
 
-        R->h_tid |= (GCFLAG_MARKED | GCFLAG_VISITED);
-
-        if (R->h_revision & 2) {
-            L = (gcptr)(R->h_revision - 2);
-            L = stmgcpage_visit(L);
-            R->h_revision = ((revision_t)L) | 2;
+        if ((R->h_original == 0) || (R->h_tid & GCFLAG_PREBUILT_ORIGINAL)) {
+            /* the obj is an original and will therefore survive: */
+            gcptr V = stmgcpage_visit(R);
+            assert(V == R);
         }
         else {
-            L = (gcptr)R->h_revision;
-            L = stmgcpage_visit(L);
-            R->h_revision = (revision_t)L;
+            assert(R->h_tid & GCFLAG_SMALLSTUB); /* only case for now */
+            /* make sure R stays valid: */
+            R->h_tid |= (GCFLAG_MARKED | GCFLAG_VISITED);
+
+            if (R->h_revision & 2) {
+                L = (gcptr)(R->h_revision - 2);
+                L = stmgcpage_visit(L);
+                R->h_revision = ((revision_t)L) | 2;
+            }
+            else {
+                L = (gcptr)R->h_revision;
+                L = stmgcpage_visit(L);
+                R->h_revision = (revision_t)L;
+            }
+
+            /* h_original will be kept up-to-date because
+               it is either == L or L's h_original. And
+               h_originals don't move */
         }
-
-        /* h_original will be kept up-to-date because
-           it is either == L or L's h_original. And
-           h_originals don't move */
     } G2L_LOOP_END;
 
 }
@@ -962,7 +992,7 @@
 
     assert(gcptrlist_size(&objects_to_trace) == 0);
     mark_prebuilt_roots();
-    mark_registered_stubs();
+    mark_registered_objs();
     mark_all_stack_roots();
     
     /* weakrefs: */
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-d92fcb9e5246
+ba0819e4b5e7

From noreply at buildbot.pypy.org  Fri Oct 25 17:52:55 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 25 Oct 2013 17:52:55 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c4: add missing string resops that really
 need barriers in stmrewrite
Message-ID: <20131025155255.E91BD1C0050@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67600:1385fb758727
Date: 2013-10-25 17:51 +0200
http://bitbucket.org/pypy/pypy/changeset/1385fb758727/

Log:	add missing string resops that really need barriers in stmrewrite

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,3 +1,18 @@
+------------------------------------------------------------
+
+constptrs always require slowpath of read_barrier if they
+point to a stub
+they also always require the slowpath of a write-barrier
+because there is always one indirection to the current version
+
+------------------------------------------------------------
+
+we may have too many transaction breaks in jitted code.
+
+------------------------------------------------------------
+
+unregister constptrs in stmgc when freeing traces
+
 ------------------------------------------------------------
 
 stm-jitdriver with autoreds
diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -69,7 +69,9 @@
             # ----------  pure operations needing read-barrier  ----------
             if opnum in (rop.GETFIELD_GC_PURE,
                         rop.GETARRAYITEM_GC_PURE,
-                        rop.ARRAYLEN_GC,):
+                        rop.ARRAYLEN_GC, rop.STRGETITEM,
+                        rop.UNICODEGETITEM, rop.STRLEN,
+                        rop.UNICODELEN):
                 # e.g. getting inst_intval of a W_IntObject that is
                 # currently only a stub needs to first resolve to a 
                 # real object
diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -646,8 +646,24 @@
             stm_transaction_break(1)
             jump()
         """)
-        py.test.skip("XXX not really right: should instead be an assert "
-                     "that p1 is already a W")
+        # py.test.skip("XXX not really right: should instead be an assert "
+        #              "that p1 is already a W")
+
+    def test_rewrite_strgetitem_unicodegetitem(self):
+        self.check_rewrite("""
+            [p1, i2, i3]
+            i4=strgetitem(p1, i2)
+            i5=unicodegetitem(p1, i2)
+            jump()
+        """, """
+            [p1, i2, i3]
+            cond_call_stm_b(p1, descr=A2Idescr)
+            i4=strgetitem(p1, i2)
+            i5=unicodegetitem(p1, i2)
+            stm_transaction_break(1)
+            jump()
+        """)
+    
 
     def test_call_release_gil(self):
         T = rffi.CArrayPtr(rffi.TIME_T)

From noreply at buildbot.pypy.org  Fri Oct 25 18:08:24 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 25 Oct 2013 18:08:24 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: don't look into this
	function
Message-ID: <20131025160824.349F81C0050@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67601:5c1b0e817070
Date: 2013-10-25 18:07 +0200
http://bitbucket.org/pypy/pypy/changeset/5c1b0e817070/

Log:	don't look into this function

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -609,6 +609,7 @@
     some += newsize >> 3
     return newsize + some
 
+ at jit.dont_look_inside
 def ll_dict_grow(d):
     if d.num_items < d.num_used_items // 4:
         ll_dict_remove_deleted_items(d)

From noreply at buildbot.pypy.org  Fri Oct 25 18:31:50 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 18:31:50 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: fijal says it's safe
Message-ID: <20131025163150.A1B661C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67602:d65280b97f33
Date: 2013-10-25 09:31 -0700
http://bitbucket.org/pypy/pypy/changeset/d65280b97f33/

Log:	fijal says it's safe

diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -87,7 +87,7 @@
         self.w_locals = w_locals
         self.locals2fast()
 
-    @jit.look_inside_iff(lambda self: jit.isvirtual(self))
+    @jit.unroll_safe
     def fast2locals(self):
         # Copy values from the fastlocals to self.w_locals
         if self.w_locals is None:
@@ -106,7 +106,7 @@
                     if not e.match(self.space, self.space.w_KeyError):
                         raise
 
-    @jit.look_inside_iff(lambda self: jit.isvirtual(self))
+    @jit.unroll_safe
     def locals2fast(self):
         # Copy values from self.w_locals to the fastlocals
         assert self.w_locals is not None

From noreply at buildbot.pypy.org  Fri Oct 25 18:40:17 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 18:40:17 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: Put is_tracefunc on the greenkey
Message-ID: <20131025164017.3A6B11C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67603:823690e51853
Date: 2013-10-25 09:39 -0700
http://bitbucket.org/pypy/pypy/changeset/823690e51853/

Log:	Put is_tracefunc on the greenkey

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -27,28 +27,35 @@
 
 JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE']
 
-def get_printable_location(next_instr, is_being_profiled, bytecode):
+def get_printable_location(next_instr, is_being_profiled, bytecode, is_tracefunc):
     from pypy.tool.stdlib_opcode import opcode_method_names
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
 
-def get_jitcell_at(next_instr, is_being_profiled, bytecode):
+def make_greenkey_dict_key(next_instr, is_being_profiled, is_tracefunc):
     # use only uints as keys in the jit_cells dict, rather than
     # a tuple (next_instr, is_being_profiled)
-    key = (next_instr << 1) | r_uint(intmask(is_being_profiled))
+    return (
+        (next_instr << 2) |
+        (r_uint(intmask(is_being_profiled)) << 1) |
+        r_uint(intmask(is_tracefunc))
+    )
+
+def get_jitcell_at(next_instr, is_being_profiled, bytecode, is_tracefunc):
+    key = make_greenkey_dict_key(next_instr, is_being_profiled, is_tracefunc)
     return bytecode.jit_cells.get(key, None)
 
-def set_jitcell_at(newcell, next_instr, is_being_profiled, bytecode):
-    key = (next_instr << 1) | r_uint(intmask(is_being_profiled))
+def set_jitcell_at(newcell, next_instr, is_being_profiled, bytecode, is_tracefunc):
+    key = make_greenkey_dict_key(next_instr, is_being_profiled, is_tracefunc)
     bytecode.jit_cells[key] = newcell
 
 
-def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
+def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode, is_tracefunc):
     return (bytecode.co_flags & CO_GENERATOR) != 0
 
 class PyPyJitDriver(JitDriver):
     reds = ['frame', 'ec']
-    greens = ['next_instr', 'is_being_profiled', 'pycode']
+    greens = ['next_instr', 'is_being_profiled', 'pycode', 'is_tracefunc']
     virtualizables = ['frame']
 
 pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,
@@ -68,7 +75,8 @@
             while True:
                 pypyjitdriver.jit_merge_point(ec=ec,
                     frame=self, next_instr=next_instr, pycode=pycode,
-                    is_being_profiled=is_being_profiled)
+                    is_being_profiled=is_being_profiled,
+                    is_tracefunc=ec.gettrace() is not None)
                 co_code = pycode.co_code
                 self.valuestackdepth = hint(self.valuestackdepth, promote=True)
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
@@ -97,7 +105,8 @@
         #
         pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
                                     pycode=self.getcode(),
-                                    is_being_profiled=self.is_being_profiled)
+                                    is_being_profiled=self.is_being_profiled,
+                                    is_tracefunc=ec.gettrace() is not None)
         return jumpto
 
 def _get_adapted_tick_counter():

From noreply at buildbot.pypy.org  Fri Oct 25 18:43:24 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 18:43:24 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: fix order
Message-ID: <20131025164324.EAE4E1C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67604:913739345d22
Date: 2013-10-25 09:42 -0700
http://bitbucket.org/pypy/pypy/changeset/913739345d22/

Log:	fix order

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -55,7 +55,7 @@
 
 class PyPyJitDriver(JitDriver):
     reds = ['frame', 'ec']
-    greens = ['next_instr', 'is_being_profiled', 'pycode', 'is_tracefunc']
+    greens = ['next_instr', 'is_being_profiled', 'is_tracefunc', 'pycode']
     virtualizables = ['frame']
 
 pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,

From noreply at buildbot.pypy.org  Fri Oct 25 18:43:26 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 18:43:26 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: order here too
Message-ID: <20131025164326.1AEC51C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67605:57961032b236
Date: 2013-10-25 09:42 -0700
http://bitbucket.org/pypy/pypy/changeset/57961032b236/

Log:	order here too

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -27,7 +27,7 @@
 
 JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE']
 
-def get_printable_location(next_instr, is_being_profiled, bytecode, is_tracefunc):
+def get_printable_location(next_instr, is_being_profiled, is_tracefunc, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
@@ -41,16 +41,16 @@
         r_uint(intmask(is_tracefunc))
     )
 
-def get_jitcell_at(next_instr, is_being_profiled, bytecode, is_tracefunc):
+def get_jitcell_at(next_instr, is_being_profiled, is_tracefunc, bytecode):
     key = make_greenkey_dict_key(next_instr, is_being_profiled, is_tracefunc)
     return bytecode.jit_cells.get(key, None)
 
-def set_jitcell_at(newcell, next_instr, is_being_profiled, bytecode, is_tracefunc):
+def set_jitcell_at(newcell, next_instr, is_being_profiled, is_tracefunc, bytecode):
     key = make_greenkey_dict_key(next_instr, is_being_profiled, is_tracefunc)
     bytecode.jit_cells[key] = newcell
 
 
-def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode, is_tracefunc):
+def should_unroll_one_iteration(next_instr, is_being_profiled, is_tracefunc, bytecode):
     return (bytecode.co_flags & CO_GENERATOR) != 0
 
 class PyPyJitDriver(JitDriver):

From noreply at buildbot.pypy.org  Fri Oct 25 19:27:34 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 19:27:34 +0200 (CEST)
Subject: [pypy-commit] pypy remove-eval-frame: Remove eval.Frame class that
 didn't add value (and added foncusion)
Message-ID: <20131025172734.AF8711C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: remove-eval-frame
Changeset: r67606:d7fd8a64c1db
Date: 2013-10-25 10:26 -0700
http://bitbucket.org/pypy/pypy/changeset/d7fd8a64c1db/

Log:	Remove eval.Frame class that didn't add value (and added foncusion)

diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -2,7 +2,7 @@
 This module defines the abstract base classes that support execution:
 Code and Frame.
 """
-from pypy.interpreter.error import OperationError
+
 from pypy.interpreter.baseobjspace import W_Root
 
 
@@ -51,88 +51,3 @@
 
     def funcrun_obj(self, func, w_obj, args):
         return self.funcrun(func, args.prepend(w_obj))
-
-
-class Frame(W_Root):
-    """A frame is an environment supporting the execution of a code object.
-    Abstract base class."""
-
-    def __init__(self, space, w_globals=None):
-        self.space = space
-        self.w_globals = w_globals  # wrapped dict of globals
-        self.w_locals = None       # wrapped dict of locals
-
-    def run(self):
-        "Abstract method to override. Runs the frame"
-        raise TypeError("abstract")
-
-    def getdictscope(self):
-        "Get the locals as a dictionary."
-        self.fast2locals()
-        return self.w_locals
-
-    def getcode(self):
-        return None
-
-    def fget_code(self, space):
-        return space.wrap(self.getcode())
-
-    def fget_getdictscope(self, space):
-        return self.getdictscope()
-
-    def setdictscope(self, w_locals):
-        "Initialize the locals from a dictionary."
-        self.w_locals = w_locals
-        self.locals2fast()
-
-    def getfastscope(self):
-        "Abstract. Get the fast locals as a list."
-        raise TypeError("abstract")
-
-    def setfastscope(self, scope_w):
-        """Abstract. Initialize the fast locals from a list of values,
-        where the order is according to self.getcode().signature()."""
-        raise TypeError("abstract")
-
-    def getfastscopelength(self):
-        "Abstract. Get the expected number of locals."
-        raise TypeError("abstract")
-
-    def fast2locals(self):
-        # Copy values from the fastlocals to self.w_locals
-        if self.w_locals is None:
-            self.w_locals = self.space.newdict()
-        varnames = self.getcode().getvarnames()
-        fastscope_w = self.getfastscope()
-        for i in range(min(len(varnames), self.getfastscopelength())):
-            name = varnames[i]
-            w_value = fastscope_w[i]
-            w_name = self.space.wrap(name)
-            if w_value is not None:
-                self.space.setitem(self.w_locals, w_name, w_value)
-            else:
-                try:
-                    self.space.delitem(self.w_locals, w_name)
-                except OperationError as e:
-                    if not e.match(self.space, self.space.w_KeyError):
-                        raise
-
-    def locals2fast(self):
-        # Copy values from self.w_locals to the fastlocals
-        assert self.w_locals is not None
-        varnames = self.getcode().getvarnames()
-        numlocals = self.getfastscopelength()
-
-        new_fastlocals_w = [None] * numlocals
-
-        for i in range(min(len(varnames), numlocals)):
-            w_name = self.space.wrap(varnames[i])
-            try:
-                w_value = self.space.getitem(self.w_locals, w_name)
-            except OperationError, e:
-                if not e.match(self.space, self.space.w_KeyError):
-                    raise
-            else:
-                new_fastlocals_w[i] = w_value
-
-        self.setfastscope(new_fastlocals_w)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -1,19 +1,19 @@
 """ PyFrame class implementation with the interpreter main loop.
 """
 
+from rpython.rlib import jit
+from rpython.rlib.debug import make_sure_not_resized, check_nonneg
+from rpython.rlib.jit import hint
+from rpython.rlib.objectmodel import we_are_translated, instantiate
+from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.tool.pairtype import extendabletype
-from pypy.interpreter import eval, pycode
+
+from pypy.interpreter import eval, pycode, pytraceback
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.executioncontext import ExecutionContext
-from pypy.interpreter import pytraceback
-from rpython.rlib.objectmodel import we_are_translated, instantiate
-from rpython.rlib.jit import hint
-from rpython.rlib.debug import make_sure_not_resized, check_nonneg
-from rpython.rlib.rarithmetic import intmask, r_uint
-from rpython.rlib import jit
 from pypy.tool import stdlib_opcode
-from rpython.tool.stdlib_opcode import host_bytecode_spec
 
 # Define some opcodes used
 for op in '''DUP_TOP POP_TOP SETUP_LOOP SETUP_EXCEPT SETUP_FINALLY
@@ -21,7 +21,8 @@
     globals()[op] = stdlib_opcode.opmap[op]
 HAVE_ARGUMENT = stdlib_opcode.HAVE_ARGUMENT
 
-class PyFrame(eval.Frame):
+
+class PyFrame(W_Root):
     """Represents a frame for a regular Python function
     that needs to be interpreted.
 
@@ -56,8 +57,10 @@
                 "use space.FrameClass(), not directly PyFrame()")
         self = hint(self, access_directly=True, fresh_virtualizable=True)
         assert isinstance(code, pycode.PyCode)
+        self.space = space
+        self.w_globals = w_globals
+        self.w_locals = None
         self.pycode = code
-        eval.Frame.__init__(self, space, w_globals)
         self.locals_stack_w = [None] * (code.co_nlocals + code.co_stacksize)
         self.valuestackdepth = code.co_nlocals
         self.lastblock = None
@@ -458,6 +461,59 @@
             self.locals_stack_w[i] = scope_w[i]
         self.init_cells()
 
+    def getdictscope(self):
+        """
+        Get the locals as a dictionary
+        """
+        self.fast2locals()
+        return self.w_locals
+
+    def setdictscope(self, w_locals):
+        """
+        Initialize the locals from a dictionary.
+        """
+        self.w_locals = w_locals
+        self.locals2fast()
+
+    def fast2locals(self):
+        # Copy values from the fastlocals to self.w_locals
+        if self.w_locals is None:
+            self.w_locals = self.space.newdict()
+        varnames = self.getcode().getvarnames()
+        fastscope_w = self.getfastscope()
+        for i in range(min(len(varnames), self.getfastscopelength())):
+            name = varnames[i]
+            w_value = fastscope_w[i]
+            w_name = self.space.wrap(name)
+            if w_value is not None:
+                self.space.setitem(self.w_locals, w_name, w_value)
+            else:
+                try:
+                    self.space.delitem(self.w_locals, w_name)
+                except OperationError as e:
+                    if not e.match(self.space, self.space.w_KeyError):
+                        raise
+
+    def locals2fast(self):
+        # Copy values from self.w_locals to the fastlocals
+        assert self.w_locals is not None
+        varnames = self.getcode().getvarnames()
+        numlocals = self.getfastscopelength()
+
+        new_fastlocals_w = [None] * numlocals
+
+        for i in range(min(len(varnames), numlocals)):
+            w_name = self.space.wrap(varnames[i])
+            try:
+                w_value = self.space.getitem(self.w_locals, w_name)
+            except OperationError, e:
+                if not e.match(self.space, self.space.w_KeyError):
+                    raise
+            else:
+                new_fastlocals_w[i] = w_value
+
+        self.setfastscope(new_fastlocals_w)
+
     def init_cells(self):
         """Initialize cellvars from self.locals_stack_w.
         This is overridden in nestedscope.py"""
@@ -475,6 +531,12 @@
     def _setcellvars(self, cellvars):
         pass
 
+    def fget_code(self, space):
+        return space.wrap(self.getcode())
+
+    def fget_getdictscope(self, space):
+        return self.getdictscope()
+
     ### line numbers ###
 
     def fget_f_lineno(self, space):
@@ -488,7 +550,7 @@
         "Returns the line number of the instruction currently being executed."
         try:
             new_lineno = space.int_w(w_new_lineno)
-        except OperationError, e:
+        except OperationError:
             raise OperationError(space.w_ValueError,
                                  space.wrap("lineno must be an integer"))
 
diff --git a/pypy/interpreter/test/test_eval.py b/pypy/interpreter/test/test_eval.py
deleted file mode 100644
--- a/pypy/interpreter/test/test_eval.py
+++ /dev/null
@@ -1,69 +0,0 @@
-
-from pypy.interpreter.eval import Frame
-from pypy.interpreter.pycode import PyCode
-
-
-class TestFrame: 
-    def setup_method(self, method):
-        def c(x, y, *args):
-            pass
-        code = PyCode._from_code(self.space, c.func_code)
-
-        class ConcreteFastscopeFrame(Frame):
-            
-            def __init__(self, space, code, numlocals):
-                self.code = code
-                Frame.__init__(self, space)
-                self.numlocals = numlocals
-                self._fastlocals_w = [None] * self.numlocals
-
-            def getcode(self):
-                return self.code
-
-            def setfastscope(self, scope_w):
-                self._fastlocals_w = scope_w
-
-            def getfastscope(self):
-                return self._fastlocals_w
-
-            def getfastscopelength(self):
-                return self.numlocals
-
-        self.f = ConcreteFastscopeFrame(self.space, code, numlocals=5)
-        
-
-    def test_fast2locals(self):
-        space = self.space 
-        w = space.wrap
-        self.f.fast2locals()
-        assert space.eq_w(self.f.w_locals, self.space.wrap({}))
-        
-        self.f._fastlocals_w[0] = w(5)
-        self.f.fast2locals()
-        assert space.eq_w(self.f.w_locals, self.space.wrap({'x': 5}))
-
-        self.f._fastlocals_w[2] = w(7)
-        self.f.fast2locals()
-        assert space.eq_w(self.f.w_locals, self.space.wrap({'x': 5, 'args': 7}))
-
-    def sameList(self, l1, l2):
-        assert len(l1) == len(l2) 
-        for w_1, w_2 in zip(l1, l2):
-            assert (w_1 is None) == (w_2 is None)
-            if w_1 is not None:
-                assert self.space.eq_w(w_1, w_2) 
-
-    def test_locals2fast(self):
-        w = self.space.wrap
-        self.f.w_locals = self.space.wrap({})
-        self.f.locals2fast()
-        self.sameList(self.f._fastlocals_w, [None]*5)
-
-        self.f.w_locals = self.space.wrap({'x': 5})
-        self.f.locals2fast()
-        self.sameList(self.f._fastlocals_w, [w(5)] + [None]*4)
-
-        self.f.w_locals = self.space.wrap({'x':5, 'args':7})
-        self.f.locals2fast()
-        self.sameList(self.f._fastlocals_w, [w(5), None, w(7),
-                                             None, None])
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -605,7 +605,7 @@
 #
 # Definition of the type's descriptors for all the internal types
 
-from pypy.interpreter.eval import Code, Frame
+from pypy.interpreter.eval import Code
 from pypy.interpreter.pycode import PyCode, CO_VARARGS, CO_VARKEYWORDS
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pyopcode import SuspendedUnroller
@@ -711,13 +711,6 @@
 BuiltinCode.typedef.acceptable_as_base_class = False
 
 
-Frame.typedef = TypeDef('internal-frame',
-    f_code = GetSetProperty(Frame.fget_code),
-    f_locals = GetSetProperty(Frame.fget_getdictscope),
-    f_globals = interp_attrproperty_w('w_globals', cls=Frame),
-    )
-Frame.typedef.acceptable_as_base_class = False
-
 PyCode.typedef = TypeDef('code',
     __new__ = interp2app(PyCode.descr_code__new__.im_func),
     __eq__ = interp2app(PyCode.descr_code__eq__),
@@ -756,7 +749,10 @@
     f_exc_value = GetSetProperty(PyFrame.fget_f_exc_value),
     f_exc_traceback = GetSetProperty(PyFrame.fget_f_exc_traceback),
     f_restricted = GetSetProperty(PyFrame.fget_f_restricted),
-    **Frame.typedef.rawdict)
+    f_code = GetSetProperty(PyFrame.fget_code),
+    f_locals = GetSetProperty(PyFrame.fget_getdictscope),
+    f_globals = interp_attrproperty_w('w_globals', cls=PyFrame),
+)
 PyFrame.typedef.acceptable_as_base_class = False
 
 Module.typedef = TypeDef("module",

From noreply at buildbot.pypy.org  Fri Oct 25 19:34:16 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 19:34:16 +0200 (CEST)
Subject: [pypy-commit] pypy remove-eval-frame: This changed
Message-ID: <20131025173416.2708A1C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: remove-eval-frame
Changeset: r67607:380243ad56c7
Date: 2013-10-25 10:33 -0700
http://bitbucket.org/pypy/pypy/changeset/380243ad56c7/

Log:	This changed

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -431,7 +431,7 @@
             i15 = getfield_gc_pure(p8, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
             i17 = int_lt(i15, 5000)
             guard_true(i17, descr=...)
-            p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.eval.Frame.inst_w_globals .*>)
+            p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_w_globals .*>)
             guard_value(p18, ConstPtr(ptr19), descr=...)
             p20 = getfield_gc(p18, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>)
             guard_value(p20, ConstPtr(ptr21), descr=...)

From noreply at buildbot.pypy.org  Fri Oct 25 19:48:45 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 25 Oct 2013 19:48:45 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: fix one benchmarks by
 tweaking the heuristic here
Message-ID: <20131025174845.182341C00D8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67608:12e93b008934
Date: 2013-10-25 19:48 +0200
http://bitbucket.org/pypy/pypy/changeset/12e93b008934/

Log:	fix one benchmarks by tweaking the heuristic here

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -611,7 +611,7 @@
 
 @jit.dont_look_inside
 def ll_dict_grow(d):
-    if d.num_items < d.num_used_items // 4:
+    if d.num_items < d.num_used_items // 2:
         ll_dict_remove_deleted_items(d)
         return True
 

From noreply at buildbot.pypy.org  Fri Oct 25 21:05:23 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 21:05:23 +0200 (CEST)
Subject: [pypy-commit] pypy remove-eval-frame: more removed instructions
Message-ID: <20131025190523.A19BE1C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: remove-eval-frame
Changeset: r67609:49c14f23008c
Date: 2013-10-25 12:03 -0700
http://bitbucket.org/pypy/pypy/changeset/49c14f23008c/

Log:	more removed instructions

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -431,7 +431,6 @@
             i15 = getfield_gc_pure(p8, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
             i17 = int_lt(i15, 5000)
             guard_true(i17, descr=...)
-            p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.pyframe.PyFrame.inst_w_globals .*>)
             guard_value(p18, ConstPtr(ptr19), descr=...)
             p20 = getfield_gc(p18, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>)
             guard_value(p20, ConstPtr(ptr21), descr=...)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_globals.py b/pypy/module/pypyjit/test_pypy_c/test_globals.py
--- a/pypy/module/pypyjit/test_pypy_c/test_globals.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_globals.py
@@ -16,8 +16,6 @@
         assert log.result == 500
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match_by_id("loadglobal", """
-            p10 = getfield_gc(p0, descr=<FieldP .*Frame.inst_w_globals .*>)
-            guard_value(p10, ConstPtr(ptr11), descr=...)
             p12 = getfield_gc(p10, descr=<FieldP .*W_DictMultiObject.inst_strategy .*>)
             guard_value(p12, ConstPtr(ptr13), descr=...)
             guard_not_invalidated(descr=...)

From noreply at buildbot.pypy.org  Fri Oct 25 21:05:24 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 21:05:24 +0200 (CEST)
Subject: [pypy-commit] pypy remove-eval-frame: Close branch for merge
Message-ID: <20131025190524.D96FE1C042B@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: remove-eval-frame
Changeset: r67610:eb0fe53861e9
Date: 2013-10-25 12:03 -0700
http://bitbucket.org/pypy/pypy/changeset/eb0fe53861e9/

Log:	Close branch for merge


From noreply at buildbot.pypy.org  Fri Oct 25 21:05:26 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 21:05:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged remove-eval-frame
Message-ID: <20131025190526.3EE281C0906@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67611:dad9dd8a5972
Date: 2013-10-25 12:04 -0700
http://bitbucket.org/pypy/pypy/changeset/dad9dd8a5972/

Log:	Merged remove-eval-frame

	Removed eval.Frame and just put all the stuff onto pyframe.PyFrame
	where it belongs. This removes confusion about which class "owns"
	the attributes, and fixes an issue where _virtualizable2_ for fields
	is marked on the wrong class.

diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -2,7 +2,7 @@
 This module defines the abstract base classes that support execution:
 Code and Frame.
 """
-from pypy.interpreter.error import OperationError
+
 from pypy.interpreter.baseobjspace import W_Root
 
 
@@ -51,88 +51,3 @@
 
     def funcrun_obj(self, func, w_obj, args):
         return self.funcrun(func, args.prepend(w_obj))
-
-
-class Frame(W_Root):
-    """A frame is an environment supporting the execution of a code object.
-    Abstract base class."""
-
-    def __init__(self, space, w_globals=None):
-        self.space = space
-        self.w_globals = w_globals  # wrapped dict of globals
-        self.w_locals = None       # wrapped dict of locals
-
-    def run(self):
-        "Abstract method to override. Runs the frame"
-        raise TypeError("abstract")
-
-    def getdictscope(self):
-        "Get the locals as a dictionary."
-        self.fast2locals()
-        return self.w_locals
-
-    def getcode(self):
-        return None
-
-    def fget_code(self, space):
-        return space.wrap(self.getcode())
-
-    def fget_getdictscope(self, space):
-        return self.getdictscope()
-
-    def setdictscope(self, w_locals):
-        "Initialize the locals from a dictionary."
-        self.w_locals = w_locals
-        self.locals2fast()
-
-    def getfastscope(self):
-        "Abstract. Get the fast locals as a list."
-        raise TypeError("abstract")
-
-    def setfastscope(self, scope_w):
-        """Abstract. Initialize the fast locals from a list of values,
-        where the order is according to self.getcode().signature()."""
-        raise TypeError("abstract")
-
-    def getfastscopelength(self):
-        "Abstract. Get the expected number of locals."
-        raise TypeError("abstract")
-
-    def fast2locals(self):
-        # Copy values from the fastlocals to self.w_locals
-        if self.w_locals is None:
-            self.w_locals = self.space.newdict()
-        varnames = self.getcode().getvarnames()
-        fastscope_w = self.getfastscope()
-        for i in range(min(len(varnames), self.getfastscopelength())):
-            name = varnames[i]
-            w_value = fastscope_w[i]
-            w_name = self.space.wrap(name)
-            if w_value is not None:
-                self.space.setitem(self.w_locals, w_name, w_value)
-            else:
-                try:
-                    self.space.delitem(self.w_locals, w_name)
-                except OperationError as e:
-                    if not e.match(self.space, self.space.w_KeyError):
-                        raise
-
-    def locals2fast(self):
-        # Copy values from self.w_locals to the fastlocals
-        assert self.w_locals is not None
-        varnames = self.getcode().getvarnames()
-        numlocals = self.getfastscopelength()
-
-        new_fastlocals_w = [None] * numlocals
-
-        for i in range(min(len(varnames), numlocals)):
-            w_name = self.space.wrap(varnames[i])
-            try:
-                w_value = self.space.getitem(self.w_locals, w_name)
-            except OperationError, e:
-                if not e.match(self.space, self.space.w_KeyError):
-                    raise
-            else:
-                new_fastlocals_w[i] = w_value
-
-        self.setfastscope(new_fastlocals_w)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -1,19 +1,19 @@
 """ PyFrame class implementation with the interpreter main loop.
 """
 
+from rpython.rlib import jit
+from rpython.rlib.debug import make_sure_not_resized, check_nonneg
+from rpython.rlib.jit import hint
+from rpython.rlib.objectmodel import we_are_translated, instantiate
+from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.tool.pairtype import extendabletype
-from pypy.interpreter import eval, pycode
+
+from pypy.interpreter import eval, pycode, pytraceback
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.executioncontext import ExecutionContext
-from pypy.interpreter import pytraceback
-from rpython.rlib.objectmodel import we_are_translated, instantiate
-from rpython.rlib.jit import hint
-from rpython.rlib.debug import make_sure_not_resized, check_nonneg
-from rpython.rlib.rarithmetic import intmask, r_uint
-from rpython.rlib import jit
 from pypy.tool import stdlib_opcode
-from rpython.tool.stdlib_opcode import host_bytecode_spec
 
 # Define some opcodes used
 for op in '''DUP_TOP POP_TOP SETUP_LOOP SETUP_EXCEPT SETUP_FINALLY
@@ -21,7 +21,8 @@
     globals()[op] = stdlib_opcode.opmap[op]
 HAVE_ARGUMENT = stdlib_opcode.HAVE_ARGUMENT
 
-class PyFrame(eval.Frame):
+
+class PyFrame(W_Root):
     """Represents a frame for a regular Python function
     that needs to be interpreted.
 
@@ -56,8 +57,10 @@
                 "use space.FrameClass(), not directly PyFrame()")
         self = hint(self, access_directly=True, fresh_virtualizable=True)
         assert isinstance(code, pycode.PyCode)
+        self.space = space
+        self.w_globals = w_globals
+        self.w_locals = None
         self.pycode = code
-        eval.Frame.__init__(self, space, w_globals)
         self.locals_stack_w = [None] * (code.co_nlocals + code.co_stacksize)
         self.valuestackdepth = code.co_nlocals
         self.lastblock = None
@@ -458,6 +461,59 @@
             self.locals_stack_w[i] = scope_w[i]
         self.init_cells()
 
+    def getdictscope(self):
+        """
+        Get the locals as a dictionary
+        """
+        self.fast2locals()
+        return self.w_locals
+
+    def setdictscope(self, w_locals):
+        """
+        Initialize the locals from a dictionary.
+        """
+        self.w_locals = w_locals
+        self.locals2fast()
+
+    def fast2locals(self):
+        # Copy values from the fastlocals to self.w_locals
+        if self.w_locals is None:
+            self.w_locals = self.space.newdict()
+        varnames = self.getcode().getvarnames()
+        fastscope_w = self.getfastscope()
+        for i in range(min(len(varnames), self.getfastscopelength())):
+            name = varnames[i]
+            w_value = fastscope_w[i]
+            w_name = self.space.wrap(name)
+            if w_value is not None:
+                self.space.setitem(self.w_locals, w_name, w_value)
+            else:
+                try:
+                    self.space.delitem(self.w_locals, w_name)
+                except OperationError as e:
+                    if not e.match(self.space, self.space.w_KeyError):
+                        raise
+
+    def locals2fast(self):
+        # Copy values from self.w_locals to the fastlocals
+        assert self.w_locals is not None
+        varnames = self.getcode().getvarnames()
+        numlocals = self.getfastscopelength()
+
+        new_fastlocals_w = [None] * numlocals
+
+        for i in range(min(len(varnames), numlocals)):
+            w_name = self.space.wrap(varnames[i])
+            try:
+                w_value = self.space.getitem(self.w_locals, w_name)
+            except OperationError, e:
+                if not e.match(self.space, self.space.w_KeyError):
+                    raise
+            else:
+                new_fastlocals_w[i] = w_value
+
+        self.setfastscope(new_fastlocals_w)
+
     def init_cells(self):
         """Initialize cellvars from self.locals_stack_w.
         This is overridden in nestedscope.py"""
@@ -475,6 +531,12 @@
     def _setcellvars(self, cellvars):
         pass
 
+    def fget_code(self, space):
+        return space.wrap(self.getcode())
+
+    def fget_getdictscope(self, space):
+        return self.getdictscope()
+
     ### line numbers ###
 
     def fget_f_lineno(self, space):
@@ -488,7 +550,7 @@
         "Returns the line number of the instruction currently being executed."
         try:
             new_lineno = space.int_w(w_new_lineno)
-        except OperationError, e:
+        except OperationError:
             raise OperationError(space.w_ValueError,
                                  space.wrap("lineno must be an integer"))
 
diff --git a/pypy/interpreter/test/test_eval.py b/pypy/interpreter/test/test_eval.py
deleted file mode 100644
--- a/pypy/interpreter/test/test_eval.py
+++ /dev/null
@@ -1,69 +0,0 @@
-
-from pypy.interpreter.eval import Frame
-from pypy.interpreter.pycode import PyCode
-
-
-class TestFrame: 
-    def setup_method(self, method):
-        def c(x, y, *args):
-            pass
-        code = PyCode._from_code(self.space, c.func_code)
-
-        class ConcreteFastscopeFrame(Frame):
-            
-            def __init__(self, space, code, numlocals):
-                self.code = code
-                Frame.__init__(self, space)
-                self.numlocals = numlocals
-                self._fastlocals_w = [None] * self.numlocals
-
-            def getcode(self):
-                return self.code
-
-            def setfastscope(self, scope_w):
-                self._fastlocals_w = scope_w
-
-            def getfastscope(self):
-                return self._fastlocals_w
-
-            def getfastscopelength(self):
-                return self.numlocals
-
-        self.f = ConcreteFastscopeFrame(self.space, code, numlocals=5)
-        
-
-    def test_fast2locals(self):
-        space = self.space 
-        w = space.wrap
-        self.f.fast2locals()
-        assert space.eq_w(self.f.w_locals, self.space.wrap({}))
-        
-        self.f._fastlocals_w[0] = w(5)
-        self.f.fast2locals()
-        assert space.eq_w(self.f.w_locals, self.space.wrap({'x': 5}))
-
-        self.f._fastlocals_w[2] = w(7)
-        self.f.fast2locals()
-        assert space.eq_w(self.f.w_locals, self.space.wrap({'x': 5, 'args': 7}))
-
-    def sameList(self, l1, l2):
-        assert len(l1) == len(l2) 
-        for w_1, w_2 in zip(l1, l2):
-            assert (w_1 is None) == (w_2 is None)
-            if w_1 is not None:
-                assert self.space.eq_w(w_1, w_2) 
-
-    def test_locals2fast(self):
-        w = self.space.wrap
-        self.f.w_locals = self.space.wrap({})
-        self.f.locals2fast()
-        self.sameList(self.f._fastlocals_w, [None]*5)
-
-        self.f.w_locals = self.space.wrap({'x': 5})
-        self.f.locals2fast()
-        self.sameList(self.f._fastlocals_w, [w(5)] + [None]*4)
-
-        self.f.w_locals = self.space.wrap({'x':5, 'args':7})
-        self.f.locals2fast()
-        self.sameList(self.f._fastlocals_w, [w(5), None, w(7),
-                                             None, None])
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -605,7 +605,7 @@
 #
 # Definition of the type's descriptors for all the internal types
 
-from pypy.interpreter.eval import Code, Frame
+from pypy.interpreter.eval import Code
 from pypy.interpreter.pycode import PyCode, CO_VARARGS, CO_VARKEYWORDS
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pyopcode import SuspendedUnroller
@@ -711,13 +711,6 @@
 BuiltinCode.typedef.acceptable_as_base_class = False
 
 
-Frame.typedef = TypeDef('internal-frame',
-    f_code = GetSetProperty(Frame.fget_code),
-    f_locals = GetSetProperty(Frame.fget_getdictscope),
-    f_globals = interp_attrproperty_w('w_globals', cls=Frame),
-    )
-Frame.typedef.acceptable_as_base_class = False
-
 PyCode.typedef = TypeDef('code',
     __new__ = interp2app(PyCode.descr_code__new__.im_func),
     __eq__ = interp2app(PyCode.descr_code__eq__),
@@ -756,7 +749,10 @@
     f_exc_value = GetSetProperty(PyFrame.fget_f_exc_value),
     f_exc_traceback = GetSetProperty(PyFrame.fget_f_exc_traceback),
     f_restricted = GetSetProperty(PyFrame.fget_f_restricted),
-    **Frame.typedef.rawdict)
+    f_code = GetSetProperty(PyFrame.fget_code),
+    f_locals = GetSetProperty(PyFrame.fget_getdictscope),
+    f_globals = interp_attrproperty_w('w_globals', cls=PyFrame),
+)
 PyFrame.typedef.acceptable_as_base_class = False
 
 Module.typedef = TypeDef("module",
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -431,7 +431,6 @@
             i15 = getfield_gc_pure(p8, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
             i17 = int_lt(i15, 5000)
             guard_true(i17, descr=...)
-            p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.eval.Frame.inst_w_globals .*>)
             guard_value(p18, ConstPtr(ptr19), descr=...)
             p20 = getfield_gc(p18, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>)
             guard_value(p20, ConstPtr(ptr21), descr=...)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_globals.py b/pypy/module/pypyjit/test_pypy_c/test_globals.py
--- a/pypy/module/pypyjit/test_pypy_c/test_globals.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_globals.py
@@ -16,8 +16,6 @@
         assert log.result == 500
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match_by_id("loadglobal", """
-            p10 = getfield_gc(p0, descr=<FieldP .*Frame.inst_w_globals .*>)
-            guard_value(p10, ConstPtr(ptr11), descr=...)
             p12 = getfield_gc(p10, descr=<FieldP .*W_DictMultiObject.inst_strategy .*>)
             guard_value(p12, ConstPtr(ptr13), descr=...)
             guard_not_invalidated(descr=...)

From noreply at buildbot.pypy.org  Fri Oct 25 21:08:51 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 21:08:51 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: merged default in
Message-ID: <20131025190851.11A491C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67612:10d4f8c79311
Date: 2013-10-25 12:06 -0700
http://bitbucket.org/pypy/pypy/changeset/10d4f8c79311/

Log:	merged default in

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -266,6 +266,7 @@
 You cannot use most existing standard library modules from RPython.  The
 exceptions are
 some functions in ``os``, ``math`` and ``time`` that have native support.
+We have our own "RPython standard library" in ``rpython.rlib.*``.
 
 To read more about the RPython limitations read the `RPython description`_.
 
diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -2,9 +2,7 @@
 This module defines the abstract base classes that support execution:
 Code and Frame.
 """
-from rpython.rlib import jit
 
-from pypy.interpreter.error import OperationError
 from pypy.interpreter.baseobjspace import W_Root
 
 
@@ -53,76 +51,3 @@
 
     def funcrun_obj(self, func, w_obj, args):
         return self.funcrun(func, args.prepend(w_obj))
-
-
-class Frame(W_Root):
-    """A frame is an environment supporting the execution of a code object.
-    Abstract base class."""
-
-    def __init__(self, space, w_globals=None):
-        self.space = space
-        self.w_globals = w_globals  # wrapped dict of globals
-        self.w_locals = None       # wrapped dict of locals
-
-    def run(self):
-        "Abstract method to override. Runs the frame"
-        raise TypeError("abstract")
-
-    def getdictscope(self):
-        "Get the locals as a dictionary."
-        self.fast2locals()
-        return self.w_locals
-
-    def getcode(self):
-        return None
-
-    def fget_code(self, space):
-        return space.wrap(self.getcode())
-
-    def fget_getdictscope(self, space):
-        return self.getdictscope()
-
-    def setdictscope(self, w_locals):
-        "Initialize the locals from a dictionary."
-        self.w_locals = w_locals
-        self.locals2fast()
-
-    @jit.unroll_safe
-    def fast2locals(self):
-        # Copy values from the fastlocals to self.w_locals
-        if self.w_locals is None:
-            self.w_locals = self.space.newdict()
-        varnames = self.getcode().getvarnames()
-        for i in range(min(len(varnames), self.getcode().co_nlocals)):
-            name = varnames[i]
-            w_value = self.locals_stack_w[i]
-            w_name = self.space.wrap(name)
-            if w_value is not None:
-                self.space.setitem(self.w_locals, w_name, w_value)
-            else:
-                try:
-                    self.space.delitem(self.w_locals, w_name)
-                except OperationError as e:
-                    if not e.match(self.space, self.space.w_KeyError):
-                        raise
-
-    @jit.unroll_safe
-    def locals2fast(self):
-        # Copy values from self.w_locals to the fastlocals
-        assert self.w_locals is not None
-        varnames = self.getcode().getvarnames()
-        numlocals = self.getcode().co_nlocals
-
-        new_fastlocals_w = [None] * numlocals
-
-        for i in range(min(len(varnames), numlocals)):
-            w_name = self.space.wrap(varnames[i])
-            try:
-                w_value = self.space.getitem(self.w_locals, w_name)
-            except OperationError, e:
-                if not e.match(self.space, self.space.w_KeyError):
-                    raise
-            else:
-                new_fastlocals_w[i] = w_value
-
-        self.setfastscope(new_fastlocals_w)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -1,19 +1,19 @@
 """ PyFrame class implementation with the interpreter main loop.
 """
 
+from rpython.rlib import jit
+from rpython.rlib.debug import make_sure_not_resized, check_nonneg
+from rpython.rlib.jit import hint
+from rpython.rlib.objectmodel import we_are_translated, instantiate
+from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.tool.pairtype import extendabletype
-from pypy.interpreter import eval, pycode
+
+from pypy.interpreter import eval, pycode, pytraceback
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.executioncontext import ExecutionContext
-from pypy.interpreter import pytraceback
-from rpython.rlib.objectmodel import we_are_translated, instantiate
-from rpython.rlib.jit import hint
-from rpython.rlib.debug import make_sure_not_resized, check_nonneg
-from rpython.rlib.rarithmetic import intmask, r_uint
-from rpython.rlib import jit
 from pypy.tool import stdlib_opcode
-from rpython.tool.stdlib_opcode import host_bytecode_spec
 
 # Define some opcodes used
 for op in '''DUP_TOP POP_TOP SETUP_LOOP SETUP_EXCEPT SETUP_FINALLY
@@ -21,7 +21,8 @@
     globals()[op] = stdlib_opcode.opmap[op]
 HAVE_ARGUMENT = stdlib_opcode.HAVE_ARGUMENT
 
-class PyFrame(eval.Frame):
+
+class PyFrame(W_Root):
     """Represents a frame for a regular Python function
     that needs to be interpreted.
 
@@ -56,8 +57,10 @@
                 "use space.FrameClass(), not directly PyFrame()")
         self = hint(self, access_directly=True, fresh_virtualizable=True)
         assert isinstance(code, pycode.PyCode)
+        self.space = space
+        self.w_globals = w_globals
+        self.w_locals = None
         self.pycode = code
-        eval.Frame.__init__(self, space, w_globals)
         self.locals_stack_w = [None] * (code.co_nlocals + code.co_stacksize)
         self.valuestackdepth = code.co_nlocals
         self.lastblock = None
@@ -453,6 +456,59 @@
             self.locals_stack_w[i] = scope_w[i]
         self.init_cells()
 
+    def getdictscope(self):
+        """
+        Get the locals as a dictionary
+        """
+        self.fast2locals()
+        return self.w_locals
+
+    def setdictscope(self, w_locals):
+        """
+        Initialize the locals from a dictionary.
+        """
+        self.w_locals = w_locals
+        self.locals2fast()
+
+    def fast2locals(self):
+        # Copy values from the fastlocals to self.w_locals
+        if self.w_locals is None:
+            self.w_locals = self.space.newdict()
+        varnames = self.getcode().getvarnames()
+        fastscope_w = self.getfastscope()
+        for i in range(min(len(varnames), self.getfastscopelength())):
+            name = varnames[i]
+            w_value = fastscope_w[i]
+            w_name = self.space.wrap(name)
+            if w_value is not None:
+                self.space.setitem(self.w_locals, w_name, w_value)
+            else:
+                try:
+                    self.space.delitem(self.w_locals, w_name)
+                except OperationError as e:
+                    if not e.match(self.space, self.space.w_KeyError):
+                        raise
+
+    def locals2fast(self):
+        # Copy values from self.w_locals to the fastlocals
+        assert self.w_locals is not None
+        varnames = self.getcode().getvarnames()
+        numlocals = self.getfastscopelength()
+
+        new_fastlocals_w = [None] * numlocals
+
+        for i in range(min(len(varnames), numlocals)):
+            w_name = self.space.wrap(varnames[i])
+            try:
+                w_value = self.space.getitem(self.w_locals, w_name)
+            except OperationError, e:
+                if not e.match(self.space, self.space.w_KeyError):
+                    raise
+            else:
+                new_fastlocals_w[i] = w_value
+
+        self.setfastscope(new_fastlocals_w)
+
     def init_cells(self):
         """Initialize cellvars from self.locals_stack_w.
         This is overridden in nestedscope.py"""
@@ -467,6 +523,12 @@
     def _setcellvars(self, cellvars):
         pass
 
+    def fget_code(self, space):
+        return space.wrap(self.getcode())
+
+    def fget_getdictscope(self, space):
+        return self.getdictscope()
+
     ### line numbers ###
 
     def fget_f_lineno(self, space):
@@ -480,7 +542,7 @@
         "Returns the line number of the instruction currently being executed."
         try:
             new_lineno = space.int_w(w_new_lineno)
-        except OperationError, e:
+        except OperationError:
             raise OperationError(space.w_ValueError,
                                  space.wrap("lineno must be an integer"))
 
diff --git a/pypy/interpreter/test/test_eval.py b/pypy/interpreter/test/test_eval.py
deleted file mode 100644
--- a/pypy/interpreter/test/test_eval.py
+++ /dev/null
@@ -1,64 +0,0 @@
-
-from pypy.interpreter.eval import Frame
-from pypy.interpreter.pycode import PyCode
-
-
-class TestFrame:
-    def setup_method(self, method):
-        def c(x, y, *args):
-            pass
-        code = PyCode._from_code(self.space, c.func_code)
-
-        class ConcreteFastscopeFrame(Frame):
-            def __init__(self, space, code, numlocals):
-                self.code = code
-                self.code.co_nlocals = numlocals
-                Frame.__init__(self, space)
-                self.locals_stack_w = [None] * numlocals
-
-            def getcode(self):
-                return self.code
-
-            def setfastscope(self, scope_w):
-                self.locals_stack_w = scope_w
-
-            def getfastscope(self):
-                return self.locals_stack_w
-
-        self.f = ConcreteFastscopeFrame(self.space, code, numlocals=5)
-
-    def test_fast2locals(self):
-        space = self.space
-        w = space.wrap
-        self.f.fast2locals()
-        assert space.eq_w(self.f.w_locals, self.space.wrap({}))
-
-        self.f.locals_stack_w[0] = w(5)
-        self.f.fast2locals()
-        assert space.eq_w(self.f.w_locals, self.space.wrap({'x': 5}))
-
-        self.f.locals_stack_w[2] = w(7)
-        self.f.fast2locals()
-        assert space.eq_w(self.f.w_locals, self.space.wrap({'x': 5, 'args': 7}))
-
-    def sameList(self, l1, l2):
-        assert len(l1) == len(l2)
-        for w_1, w_2 in zip(l1, l2):
-            assert (w_1 is None) == (w_2 is None)
-            if w_1 is not None:
-                assert self.space.eq_w(w_1, w_2)
-
-    def test_locals2fast(self):
-        w = self.space.wrap
-        self.f.w_locals = self.space.wrap({})
-        self.f.locals2fast()
-        self.sameList(self.f.locals_stack_w, [None] * 5)
-
-        self.f.w_locals = self.space.wrap({'x': 5})
-        self.f.locals2fast()
-        self.sameList(self.f.locals_stack_w, [w(5)] + [None] * 4)
-
-        self.f.w_locals = self.space.wrap({'x': 5, 'args': 7})
-        self.f.locals2fast()
-        self.sameList(self.f.locals_stack_w, [w(5), None, w(7),
-                                             None, None])
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -605,7 +605,7 @@
 #
 # Definition of the type's descriptors for all the internal types
 
-from pypy.interpreter.eval import Code, Frame
+from pypy.interpreter.eval import Code
 from pypy.interpreter.pycode import PyCode, CO_VARARGS, CO_VARKEYWORDS
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pyopcode import SuspendedUnroller
@@ -711,13 +711,6 @@
 BuiltinCode.typedef.acceptable_as_base_class = False
 
 
-Frame.typedef = TypeDef('internal-frame',
-    f_code = GetSetProperty(Frame.fget_code),
-    f_locals = GetSetProperty(Frame.fget_getdictscope),
-    f_globals = interp_attrproperty_w('w_globals', cls=Frame),
-    )
-Frame.typedef.acceptable_as_base_class = False
-
 PyCode.typedef = TypeDef('code',
     __new__ = interp2app(PyCode.descr_code__new__.im_func),
     __eq__ = interp2app(PyCode.descr_code__eq__),
@@ -756,7 +749,10 @@
     f_exc_value = GetSetProperty(PyFrame.fget_f_exc_value),
     f_exc_traceback = GetSetProperty(PyFrame.fget_f_exc_traceback),
     f_restricted = GetSetProperty(PyFrame.fget_f_restricted),
-    **Frame.typedef.rawdict)
+    f_code = GetSetProperty(PyFrame.fget_code),
+    f_locals = GetSetProperty(PyFrame.fget_getdictscope),
+    f_globals = interp_attrproperty_w('w_globals', cls=PyFrame),
+)
 PyFrame.typedef.acceptable_as_base_class = False
 
 Module.typedef = TypeDef("module",
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -207,7 +207,7 @@
 
     def unpack_list_of_int_items(self, w_cdata):
         if self.size == rffi.sizeof(rffi.LONG):
-            from rpython.rlib.rarray import populate_list_from_raw_array
+            from rpython.rlib.rrawarray import populate_list_from_raw_array
             res = []
             buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
             length = w_cdata.get_array_length()
@@ -223,7 +223,7 @@
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
             if self.size == rffi.sizeof(rffi.LONG): # fastest path
-                from rpython.rlib.rarray import copy_list_to_raw_array
+                from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
@@ -367,7 +367,7 @@
 
     def unpack_list_of_float_items(self, w_cdata):
         if self.size == rffi.sizeof(rffi.DOUBLE):
-            from rpython.rlib.rarray import populate_list_from_raw_array
+            from rpython.rlib.rrawarray import populate_list_from_raw_array
             res = []
             buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
             length = w_cdata.get_array_length()
@@ -383,7 +383,7 @@
         float_list = self.space.listview_float(w_ob)
         if float_list is not None:
             if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
-                from rpython.rlib.rarray import copy_list_to_raw_array
+                from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.DOUBLEP, cdata)
                 copy_list_to_raw_array(float_list, cdata)
                 return True
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -136,19 +136,19 @@
 
     def setup_method(self, meth):
         from pypy.interpreter import gateway
-        from rpython.rlib import rarray
+        from rpython.rlib import rrawarray
         #
         self.count = 0
         def get_count(*args):
             return self.space.wrap(self.count)
         self.w_get_count = self.space.wrap(gateway.interp2app(get_count))
         #
-        original = rarray.populate_list_from_raw_array
+        original = rrawarray.populate_list_from_raw_array
         def populate_list_from_raw_array(*args):
             self.count += 1
             return original(*args)
         self._original = original
-        rarray.populate_list_from_raw_array = populate_list_from_raw_array
+        rrawarray.populate_list_from_raw_array = populate_list_from_raw_array
         #
         original2 = misc.unpack_list_from_raw_array
         def unpack_list_from_raw_array(*args):
@@ -177,8 +177,8 @@
 
 
     def teardown_method(self, meth):
-        from rpython.rlib import rarray
-        rarray.populate_list_from_raw_array = self._original
+        from rpython.rlib import rrawarray
+        rrawarray.populate_list_from_raw_array = self._original
         misc.unpack_list_from_raw_array = self._original2
         misc.unpack_cfloat_list_from_raw_array = self._original3
         misc.unpack_unsigned_list_from_raw_array = self._original4
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -431,7 +431,6 @@
             i15 = getfield_gc_pure(p8, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
             i17 = int_lt(i15, 5000)
             guard_true(i17, descr=...)
-            p18 = getfield_gc(p0, descr=<FieldP pypy.interpreter.eval.Frame.inst_w_globals .*>)
             guard_value(p18, ConstPtr(ptr19), descr=...)
             p20 = getfield_gc(p18, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>)
             guard_value(p20, ConstPtr(ptr21), descr=...)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_globals.py b/pypy/module/pypyjit/test_pypy_c/test_globals.py
--- a/pypy/module/pypyjit/test_pypy_c/test_globals.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_globals.py
@@ -16,8 +16,6 @@
         assert log.result == 500
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match_by_id("loadglobal", """
-            p10 = getfield_gc(p0, descr=<FieldP .*Frame.inst_w_globals .*>)
-            guard_value(p10, ConstPtr(ptr11), descr=...)
             p12 = getfield_gc(p10, descr=<FieldP .*W_DictMultiObject.inst_strategy .*>)
             guard_value(p12, ConstPtr(ptr13), descr=...)
             guard_not_invalidated(descr=...)
diff --git a/rpython/rlib/rarray.py b/rpython/rlib/rrawarray.py
rename from rpython/rlib/rarray.py
rename to rpython/rlib/rrawarray.py
--- a/rpython/rlib/rarray.py
+++ b/rpython/rlib/rrawarray.py
@@ -1,5 +1,4 @@
 from rpython.annotator import model as annmodel
-from rpython.annotator.listdef import ListDef
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, llmemory
diff --git a/rpython/rlib/test/test_rarray.py b/rpython/rlib/test/test_rrawarray.py
rename from rpython/rlib/test/test_rarray.py
rename to rpython/rlib/test/test_rrawarray.py
--- a/rpython/rlib/test/test_rarray.py
+++ b/rpython/rlib/test/test_rrawarray.py
@@ -1,4 +1,5 @@
-from rpython.rlib.rarray import copy_list_to_raw_array, populate_list_from_raw_array
+from rpython.rlib.rrawarray import copy_list_to_raw_array, \
+                                   populate_list_from_raw_array
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.test.tool import BaseRtypingTest
 

From noreply at buildbot.pypy.org  Fri Oct 25 21:08:52 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 21:08:52 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: Moved changes over
Message-ID: <20131025190852.6539B1C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67613:acdbfb92636b
Date: 2013-10-25 12:08 -0700
http://bitbucket.org/pypy/pypy/changeset/acdbfb92636b/

Log:	Moved changes over

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -8,7 +8,7 @@
 from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.tool.pairtype import extendabletype
 
-from pypy.interpreter import eval, pycode, pytraceback
+from pypy.interpreter import pycode, pytraceback
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, operationerrfmt
@@ -470,15 +470,15 @@
         self.w_locals = w_locals
         self.locals2fast()
 
+    @jit.unroll_safe
     def fast2locals(self):
         # Copy values from the fastlocals to self.w_locals
         if self.w_locals is None:
             self.w_locals = self.space.newdict()
         varnames = self.getcode().getvarnames()
-        fastscope_w = self.getfastscope()
-        for i in range(min(len(varnames), self.getfastscopelength())):
+        for i in range(min(len(varnames), self.getcode().co_nlocals)):
             name = varnames[i]
-            w_value = fastscope_w[i]
+            w_value = self.locals_stack_w[i]
             w_name = self.space.wrap(name)
             if w_value is not None:
                 self.space.setitem(self.w_locals, w_name, w_value)
@@ -489,11 +489,12 @@
                     if not e.match(self.space, self.space.w_KeyError):
                         raise
 
+    @jit.unroll_safe
     def locals2fast(self):
         # Copy values from self.w_locals to the fastlocals
         assert self.w_locals is not None
         varnames = self.getcode().getvarnames()
-        numlocals = self.getfastscopelength()
+        numlocals = self.getcode().co_nlocals
 
         new_fastlocals_w = [None] * numlocals
 

From noreply at buildbot.pypy.org  Fri Oct 25 21:15:36 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 21:15:36 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: these should have been set as well:
Message-ID: <20131025191536.9DC3F1C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67614:74c7fe24d0eb
Date: 2013-10-25 12:15 -0700
http://bitbucket.org/pypy/pypy/changeset/74c7fe24d0eb/

Log:	these should have been set as well:

diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -127,7 +127,7 @@
                                  self.space.wrap("bad cellvars"))
         self.cells[:ncellvars] = cellvars
 
-    @jit.look_inside_iff(lambda self: jit.isvirtual(self))
+    @jit.unroll_safe
     def fast2locals(self):
         super_fast2locals(self)
         # cellvars are values exported to inner scopes
@@ -146,7 +146,7 @@
                 w_name = self.space.wrap(name)
                 self.space.setitem(self.w_locals, w_name, w_value)
 
-    @jit.look_inside_iff(lambda self: jit.isvirtual(self))
+    @jit.unroll_safe
     def locals2fast(self):
         super_locals2fast(self)
         freevarnames = self.pycode.co_cellvars + self.pycode.co_freevars

From noreply at buildbot.pypy.org  Fri Oct 25 21:17:22 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 25 Oct 2013 21:17:22 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: not sure how this sneaked in
Message-ID: <20131025191722.A47591C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67615:ad372032051b
Date: 2013-10-25 12:16 -0700
http://bitbucket.org/pypy/pypy/changeset/ad372032051b/

Log:	not sure how this sneaked in

diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -39,7 +39,6 @@
         #
         for index, jd in enumerate(jitdrivers_sd):
             jd.index = index
-        self.seen = DependencyTracker(self.readwrite_analyzer)
 
     def find_all_graphs(self, policy):
         try:

From noreply at buildbot.pypy.org  Fri Oct 25 22:12:18 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 25 Oct 2013 22:12:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: small cleanups
Message-ID: <20131025201218.B59891C019E@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67616:ff15fb05ce74
Date: 2013-10-25 16:11 -0400
http://bitbucket.org/pypy/pypy/changeset/ff15fb05ce74/

Log:	small cleanups

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -166,10 +166,9 @@
                 self._prepare_array_index(space, w_index)
         if iter_shape is None:
             # w_index is a list of slices
-            w_value = convert_to_array(space, w_value)
             chunks = self.implementation._prepare_slice_args(space, w_index)
             view = chunks.apply(space, self)
-            view.implementation.setslice(space, w_value)
+            view.implementation.setslice(space, val_arr)
             return
         loop.setitem_array_int(space, self, iter_shape, indexes, val_arr,
                                prefix)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -58,8 +58,6 @@
     specialize.argtype(1)(func)
     @functools.wraps(func)
     def dispatcher(self, v):
-        from pypy.module.micronumpy.interp_boxes import W_GenericBox
-        assert isinstance(v, W_GenericBox)
         return self.box_component(
             func(
                 self,

From noreply at buildbot.pypy.org  Fri Oct 25 23:23:19 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 25 Oct 2013 23:23:19 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: Test and a fix and tweak a
	heuristic
Message-ID: <20131025212319.798901C103B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67617:78295af62ea0
Date: 2013-10-25 23:22 +0200
http://bitbucket.org/pypy/pypy/changeset/78295af62ea0/

Log:	Test and a fix and tweak a heuristic

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -611,7 +611,8 @@
 
 @jit.dont_look_inside
 def ll_dict_grow(d):
-    if d.num_items < d.num_used_items // 2:
+    # don't reindex the dict if it's tiny
+    if d.num_items < d.num_used_items // 2 and d.num_items >= 32:
         ll_dict_remove_deleted_items(d)
         return True
 
@@ -639,7 +640,7 @@
     ENTRY = lltype.typeOf(d).TO.entries.TO.OF
     isrc = 0
     idst = 0
-    while isrc < len(d.entries):
+    while isrc < d.num_used_items:
         if d.entries.valid(isrc):
             src = d.entries[isrc]
             dst = newitems[idst]
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -136,10 +136,10 @@
         DICT = self._get_str_dict()
         ll_d = rdict.ll_newdict(DICT)
         lls = llstr("a")
-        for i in range(20):
+        for i in range(40):
             rdict.ll_dict_setitem(ll_d, lls, i)
             rdict.ll_dict_delitem(ll_d, lls)
-        assert ll_d.num_used_items <= 4
+        assert ll_d.num_used_items <= 10
 
     def test_dict_iteration(self):
         DICT = self._get_str_dict()
@@ -272,6 +272,22 @@
 
 class TestRdict(BaseRtypingTest):
 
+    def test_bug(self):
+        keys = [str(i) for i in range(0, 100, 2)]
+
+        def f():
+            d = {}
+            for key in keys:
+                d[key] = 0
+            for i in range(4):
+                print i
+                d['0'] = 13
+                for j in range(len(keys) - 1):
+                    del d[keys[j]]
+                    d[keys[j + 1]] = 42
+
+        self.interpret(f, [])
+
     def test_dict_creation(self):
         def createdict(i):
             d = {'hello' : i}

From noreply at buildbot.pypy.org  Fri Oct 25 23:24:04 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 25 Oct 2013 23:24:04 +0200 (CEST)
Subject: [pypy-commit] pypy rdict-experiments-3: ups tweak this back
Message-ID: <20131025212404.37B061C103B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rdict-experiments-3
Changeset: r67618:a0be44456d2c
Date: 2013-10-25 23:23 +0200
http://bitbucket.org/pypy/pypy/changeset/a0be44456d2c/

Log:	ups tweak this back

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -612,7 +612,7 @@
 @jit.dont_look_inside
 def ll_dict_grow(d):
     # don't reindex the dict if it's tiny
-    if d.num_items < d.num_used_items // 2 and d.num_items >= 32:
+    if d.num_items < d.num_used_items // 2 and d.num_used_items >= 64:
         ll_dict_remove_deleted_items(d)
         return True
 

From noreply at buildbot.pypy.org  Sat Oct 26 00:36:41 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 00:36:41 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: update things,
 on further review we don't want this though
Message-ID: <20131025223641.D41D01C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67619:2170c11c35e6
Date: 2013-10-25 14:27 -0700
http://bitbucket.org/pypy/pypy/changeset/2170c11c35e6/

Log:	update things, on further review we don't want this though

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -27,35 +27,34 @@
 
 JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE']
 
-def get_printable_location(next_instr, is_being_profiled, is_tracefunc, bytecode):
+def get_printable_location(next_instr, is_being_profiled, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
 
-def make_greenkey_dict_key(next_instr, is_being_profiled, is_tracefunc):
+def make_greenkey_dict_key(next_instr, is_being_profiled):
     # use only uints as keys in the jit_cells dict, rather than
     # a tuple (next_instr, is_being_profiled)
     return (
-        (next_instr << 2) |
-        (r_uint(intmask(is_being_profiled)) << 1) |
-        r_uint(intmask(is_tracefunc))
+        (next_instr << 1) |
+        r_uint(intmask(is_being_profiled))
     )
 
-def get_jitcell_at(next_instr, is_being_profiled, is_tracefunc, bytecode):
-    key = make_greenkey_dict_key(next_instr, is_being_profiled, is_tracefunc)
+def get_jitcell_at(next_instr, is_being_profiled, bytecode):
+    key = make_greenkey_dict_key(next_instr, is_being_profiled)
     return bytecode.jit_cells.get(key, None)
 
-def set_jitcell_at(newcell, next_instr, is_being_profiled, is_tracefunc, bytecode):
-    key = make_greenkey_dict_key(next_instr, is_being_profiled, is_tracefunc)
+def set_jitcell_at(newcell, next_instr, is_being_profiled, bytecode):
+    key = make_greenkey_dict_key(next_instr, is_being_profiled)
     bytecode.jit_cells[key] = newcell
 
 
-def should_unroll_one_iteration(next_instr, is_being_profiled, is_tracefunc, bytecode):
+def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
     return (bytecode.co_flags & CO_GENERATOR) != 0
 
 class PyPyJitDriver(JitDriver):
     reds = ['frame', 'ec']
-    greens = ['next_instr', 'is_being_profiled', 'is_tracefunc', 'pycode']
+    greens = ['next_instr', 'is_being_profiled', 'pycode']
     virtualizables = ['frame']
 
 pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,
@@ -75,8 +74,7 @@
             while True:
                 pypyjitdriver.jit_merge_point(ec=ec,
                     frame=self, next_instr=next_instr, pycode=pycode,
-                    is_being_profiled=is_being_profiled,
-                    is_tracefunc=ec.gettrace() is not None)
+                    is_being_profiled=is_being_profiled)
                 co_code = pycode.co_code
                 self.valuestackdepth = hint(self.valuestackdepth, promote=True)
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
@@ -105,8 +103,7 @@
         #
         pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
                                     pycode=self.getcode(),
-                                    is_being_profiled=self.is_being_profiled,
-                                    is_tracefunc=ec.gettrace() is not None)
+                                    is_being_profiled=self.is_being_profiled)
         return jumpto
 
 def _get_adapted_tick_counter():
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -67,7 +67,7 @@
         # LOAD_GLOBAL of OFFSET
         ops = entry_bridge.ops_by_id('cond', opcode='LOAD_GLOBAL')
         assert log.opnames(ops) == ["guard_value",
-                                    "getfield_gc", "guard_value",
+                                    "guard_value",
                                     "getfield_gc", "guard_value",
                                     "guard_not_invalidated"]
         ops = entry_bridge.ops_by_id('add', opcode='LOAD_GLOBAL')
@@ -81,7 +81,7 @@
             p39 = getfield_gc(p38, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
             i40 = force_token()
             p41 = getfield_gc(p38, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
-            guard_isnull(p41, descr=...)
+            guard_value(p41, ConstPtr(ptr42), descr=...)
             i42 = getfield_gc(p38, descr=<FieldU pypy.interpreter.executioncontext.ExecutionContext.inst_profilefunc .*>)
             i43 = int_is_zero(i42)
             guard_true(i43, descr=...)
@@ -424,33 +424,34 @@
             """, [])
         loop, = log.loops_by_id('call', is_entry_bridge=True)
         assert loop.match("""
-            guard_value(i6, 1, descr=...)
-            guard_nonnull_class(p8, ConstClass(W_IntObject), descr=...)
-            guard_value(i4, 0, descr=...)
-            guard_value(p3, ConstPtr(ptr14), descr=...)
-            i15 = getfield_gc_pure(p8, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
-            i17 = int_lt(i15, 5000)
-            guard_true(i17, descr=...)
-            guard_value(p18, ConstPtr(ptr19), descr=...)
-            p20 = getfield_gc(p18, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>)
+            guard_value(i4, 1, descr=...)
+            guard_nonnull_class(p12, ConstClass(W_IntObject), descr=...)
+            p20 = getfield_gc(p1, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
             guard_value(p20, ConstPtr(ptr21), descr=...)
+            guard_value(i8, 0, descr=...)
+            guard_value(p2, ConstPtr(ptr23), descr=...)
+            i24 = getfield_gc_pure(p12, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
+            i26 = int_lt(i24, 5000)
+            guard_true(i26, descr=...)
+            guard_value(p7, ConstPtr(ptr27), descr=...)
+            p28 = getfield_gc(p7, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>)
+            guard_value(p28, ConstPtr(ptr29), descr=...)
             guard_not_invalidated(descr=...)
-            # most importantly, there is no getarrayitem_gc here
-            p23 = call(ConstClass(getexecutioncontext), descr=<Callr . EF=1>)
-            p24 = getfield_gc(p23, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
-            i25 = force_token()
-            p26 = getfield_gc(p23, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
-            guard_isnull(p26, descr=...)
-            i27 = getfield_gc(p23, descr=<FieldU pypy.interpreter.executioncontext.ExecutionContext.inst_profilefunc .*>)
-            i28 = int_is_zero(i27)
-            guard_true(i28, descr=...)
-            p30 = getfield_gc(ConstPtr(ptr29), descr=<FieldP pypy.interpreter.nestedscope.Cell.inst_w_value .*>)
-            guard_nonnull_class(p30, ConstClass(W_IntObject), descr=...)
-            i32 = getfield_gc_pure(p30, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
-            i33 = int_add_ovf(i15, i32)
+            p31 = call(ConstClass(getexecutioncontext), descr=<Callr 8 EF=1>)
+            p32 = getfield_gc(p31, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
+            p33 = force_token()
+            p34 = getfield_gc(p31, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
+            guard_value(p34, ConstPtr(ptr35), descr=...)
+            i36 = getfield_gc(p31, descr=<FieldU pypy.interpreter.executioncontext.ExecutionContext.inst_profilefunc .*>)
+            i37 = int_is_zero(i36)
+            guard_true(i37, descr=...)
+            p39 = getfield_gc(ConstPtr(ptr38), descr=<FieldP pypy.interpreter.nestedscope.Cell.inst_w_value .*>)
+            guard_nonnull_class(p39, ConstClass(W_IntObject), descr=...)
+            i41 = getfield_gc_pure(p39, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
+            i42 = int_add_ovf(i24, i41)
             guard_no_overflow(descr=...)
+            guard_not_invalidated(descr=...)
             --TICK--
-            p39 = same_as(...) # Should be killed by backend
         """)
 
     def test_local_closure_is_virtual(self):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -60,29 +60,33 @@
         assert log.result == main(500)
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-        i55 = int_gt(i43, 0)
-        guard_true(i55, descr=...)
-        p56 = force_token()
-        setfield_gc(p0, p56, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
-        i57 = call_release_gil(..., i36, 1, descr=<Calli 4 ii EF=6>)
+        i63 = int_gt(i48, 0)
+        guard_true(i63, descr=...)
+        p64 = force_token()
+        setfield_gc(p0, p64, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        i65 = call_release_gil(..., i39, 1, descr=<Calli 4 ii EF=6>)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
-        i58 = int_is_true(i57)
-        guard_true(i58, descr=...)
-        i59 = int_sub(i43, 1)
+        i66 = int_is_true(i65)
+        guard_true(i66, descr=...)
+        p67 = getfield_gc(p1, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
+        guard_value(p67, ConstPtr(ptr46), descr=...)
+        i68 = int_sub(i48, 1)
         guard_not_invalidated(descr=...)
-        p61 = force_token()
-        setfield_gc(p0, p61, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
-        i62 = call_release_gil(..., i36, 0, descr=<Calli 4 ii EF=6>)
+        p69 = force_token()
+        setfield_gc(p0, p69, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        i70 = call_release_gil(..., i39, 0, descr=<Calli 4 ii EF=6>)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
-        i63 = int_is_true(i62)
-        guard_false(i63, descr=...)
-        p64 = force_token()
-        setfield_gc(p0, p64, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
-        call_release_gil(..., i36, descr=<Callv 0 i EF=6>)
+        i71 = int_is_true(i70)
+        guard_false(i71, descr=...)
+        p72 = force_token()
+        setfield_gc(p0, p72, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
+        call_release_gil(..., i39, descr=<Callv 0 i EF=6>)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
+        p73 = getfield_gc(p1, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
+        guard_value(p73, ConstPtr(ptr58), descr=...)
         guard_not_invalidated(descr=...)
         --TICK--
         jump(..., descr=...)

From noreply at buildbot.pypy.org  Sat Oct 26 00:36:43 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 00:36:43 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: Updated pypyjit for new code
Message-ID: <20131025223643.1973F1C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67620:7ec5aa732241
Date: 2013-10-25 15:34 -0700
http://bitbucket.org/pypy/pypy/changeset/7ec5aa732241/

Log:	Updated pypyjit for new code

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -426,29 +426,27 @@
         assert loop.match("""
             guard_value(i4, 1, descr=...)
             guard_nonnull_class(p12, ConstClass(W_IntObject), descr=...)
-            p20 = getfield_gc(p1, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
-            guard_value(p20, ConstPtr(ptr21), descr=...)
             guard_value(i8, 0, descr=...)
-            guard_value(p2, ConstPtr(ptr23), descr=...)
-            i24 = getfield_gc_pure(p12, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
-            i26 = int_lt(i24, 5000)
-            guard_true(i26, descr=...)
-            guard_value(p7, ConstPtr(ptr27), descr=...)
-            p28 = getfield_gc(p7, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>)
-            guard_value(p28, ConstPtr(ptr29), descr=...)
+            guard_value(p2, ConstPtr(ptr21), descr=...)
+            i22 = getfield_gc_pure(p12, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
+            i24 = int_lt(i22, 5000)
+            guard_true(i24, descr=...)
+            guard_value(p7, ConstPtr(ptr25), descr=...)
+            p26 = getfield_gc(p7, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>)
+            guard_value(p26, ConstPtr(ptr27), descr=...)
             guard_not_invalidated(descr=...)
-            p31 = call(ConstClass(getexecutioncontext), descr=<Callr 8 EF=1>)
-            p32 = getfield_gc(p31, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
-            p33 = force_token()
-            p34 = getfield_gc(p31, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
-            guard_value(p34, ConstPtr(ptr35), descr=...)
-            i36 = getfield_gc(p31, descr=<FieldU pypy.interpreter.executioncontext.ExecutionContext.inst_profilefunc .*>)
-            i37 = int_is_zero(i36)
-            guard_true(i37, descr=...)
-            p39 = getfield_gc(ConstPtr(ptr38), descr=<FieldP pypy.interpreter.nestedscope.Cell.inst_w_value .*>)
-            guard_nonnull_class(p39, ConstClass(W_IntObject), descr=...)
-            i41 = getfield_gc_pure(p39, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
-            i42 = int_add_ovf(i24, i41)
+            p29 = call(ConstClass(getexecutioncontext), descr=<Callr 8 EF=1>)
+            p30 = getfield_gc(p29, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
+            p31 = force_token()
+            p32 = getfield_gc(p29, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
+            guard_value(p32, ConstPtr(ptr33), descr=...)
+            i34 = getfield_gc(p29, descr=<FieldU pypy.interpreter.executioncontext.ExecutionContext.inst_profilefunc .*>)
+            i35 = int_is_zero(i34)
+            guard_true(i35, descr=...)
+            p37 = getfield_gc(ConstPtr(ptr36), descr=<FieldP pypy.interpreter.nestedscope.Cell.inst_w_value .*>)
+            guard_nonnull_class(p37, ConstClass(W_IntObject), descr=...)
+            i39 = getfield_gc_pure(p37, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
+            i40 = int_add_ovf(i22, i39)
             guard_no_overflow(descr=...)
             guard_not_invalidated(descr=...)
             --TICK--
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -60,33 +60,29 @@
         assert log.result == main(500)
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-        i63 = int_gt(i48, 0)
-        guard_true(i63, descr=...)
-        p64 = force_token()
-        setfield_gc(p0, p64, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
-        i65 = call_release_gil(..., i39, 1, descr=<Calli 4 ii EF=6>)
+        i56 = int_gt(i44, 0)
+        guard_true(i56, descr=...)
+        p57 = force_token()
+        setfield_gc(p0, p57, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 8>)
+        i58 = call_release_gil(..., i37, 1, descr=<Calli 4 ii EF=6>)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
-        i66 = int_is_true(i65)
-        guard_true(i66, descr=...)
-        p67 = getfield_gc(p1, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
-        guard_value(p67, ConstPtr(ptr46), descr=...)
-        i68 = int_sub(i48, 1)
+        i59 = int_is_true(i58)
+        guard_true(i59, descr=...)
+        i60 = int_sub(i44, 1)
         guard_not_invalidated(descr=...)
-        p69 = force_token()
-        setfield_gc(p0, p69, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
-        i70 = call_release_gil(..., i39, 0, descr=<Calli 4 ii EF=6>)
+        p62 = force_token()
+        setfield_gc(p0, p62, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 8>)
+        i63 = call_release_gil(..., i37, 0, descr=<Calli 4 ii EF=6>)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
-        i71 = int_is_true(i70)
-        guard_false(i71, descr=...)
-        p72 = force_token()
-        setfield_gc(p0, p72, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
-        call_release_gil(..., i39, descr=<Callv 0 i EF=6>)
+        i64 = int_is_true(i63)
+        guard_false(i64, descr=...)
+        p65 = force_token()
+        setfield_gc(p0, p65, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 8>)
+        call_release_gil(..., i37, descr=<Callv 0 i EF=6>)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
-        p73 = getfield_gc(p1, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
-        guard_value(p73, ConstPtr(ptr58), descr=...)
         guard_not_invalidated(descr=...)
         --TICK--
         jump(..., descr=...)

From noreply at buildbot.pypy.org  Sat Oct 26 00:36:44 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 00:36:44 +0200 (CEST)
Subject: [pypy-commit] pypy jit-settrace: CLosing branch for merge
Message-ID: <20131025223644.33D711C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-settrace
Changeset: r67621:e2533214b285
Date: 2013-10-25 15:35 -0700
http://bitbucket.org/pypy/pypy/changeset/e2533214b285/

Log:	CLosing branch for merge


From noreply at buildbot.pypy.org  Sat Oct 26 00:36:45 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 00:36:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge jit-settrace.
Message-ID: <20131025223645.EB5711C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67622:6b62779684ac
Date: 2013-10-25 15:35 -0700
http://bitbucket.org/pypy/pypy/changeset/6b62779684ac/

Log:	Merge jit-settrace.

	This lets the JIT stay active when sys.settrace() has a tracefunc
	enabled

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -76,7 +76,7 @@
                 frame_vref()
             jit.virtual_ref_finish(frame_vref, frame)
 
-        if self.w_tracefunc is not None and not frame.hide():
+        if self.gettrace() is not None and not frame.hide():
             self.space.frame_trace_action.fire()
 
     # ________________________________________________________________
@@ -115,14 +115,14 @@
 
     def call_trace(self, frame):
         "Trace the call of a function"
-        if self.w_tracefunc is not None or self.profilefunc is not None:
+        if self.gettrace() is not None or self.profilefunc is not None:
             self._trace(frame, 'call', self.space.w_None)
             if self.profilefunc:
                 frame.is_being_profiled = True
 
     def return_trace(self, frame, w_retval):
         "Trace the return from a function"
-        if self.w_tracefunc is not None:
+        if self.gettrace() is not None:
             return_from_hidden = self._trace(frame, 'return', w_retval)
             # special case: if we are returning from a hidden function,
             # then maybe we have to fire() the action again; otherwise
@@ -152,7 +152,7 @@
     def exception_trace(self, frame, operationerr):
         "Trace function called upon OperationError."
         operationerr.record_interpreter_traceback()
-        if self.w_tracefunc is not None:
+        if self.gettrace() is not None:
             self._trace(frame, 'exception', None, operationerr)
         #operationerr.print_detailed_traceback(self.space)
 
@@ -181,7 +181,7 @@
             self.space.frame_trace_action.fire()
 
     def gettrace(self):
-        return self.w_tracefunc
+        return jit.promote(self.w_tracefunc)
 
     def setprofile(self, w_func):
         """Set the global trace function."""
@@ -234,7 +234,7 @@
 
         # Tracing cases
         if event == 'call':
-            w_callback = self.w_tracefunc
+            w_callback = self.gettrace()
         else:
             w_callback = frame.w_f_trace
 
@@ -310,6 +310,9 @@
     the GIL.  And whether we have threads or not, it is forced to zero
     whenever we fire any of the asynchronous actions.
     """
+
+    _immutable_fields_ = ["checkinterval_scaled?"]
+
     def __init__(self):
         self._periodic_actions = []
         self._nonperiodic_actions = []
@@ -367,7 +370,7 @@
     def _rebuild_action_dispatcher(self):
         periodic_actions = unrolling_iterable(self._periodic_actions)
 
-        @jit.dont_look_inside
+        @jit.unroll_safe
         def action_dispatcher(ec, frame):
             # periodic actions (first reset the bytecode counter)
             self.reset_ticker(self.checkinterval_scaled)
@@ -454,6 +457,9 @@
     def perform(self, executioncontext, frame):
         if self.finalizers_lock_count > 0:
             return
+        self._run_finalizers()
+
+    def _run_finalizers(self):
         # Each call to perform() first grabs the self.dying_objects
         # and replaces it with an empty list.  We do this to try to
         # avoid too deep recursions of the kind of __del__ being called
@@ -473,9 +479,10 @@
 class FrameTraceAction(AsyncAction):
     """An action that calls the local trace functions (w_f_trace)."""
 
+    @jit.unroll_safe
     def perform(self, executioncontext, frame):
         if (frame.w_f_trace is None or executioncontext.is_tracing or
-            executioncontext.w_tracefunc is None):
+            executioncontext.gettrace() is None):
             return
         code = frame.pycode
         if frame.instr_lb <= frame.last_instr < frame.instr_ub:
diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -127,7 +127,7 @@
                                  self.space.wrap("bad cellvars"))
         self.cells[:ncellvars] = cellvars
 
-    @jit.dont_look_inside
+    @jit.unroll_safe
     def fast2locals(self):
         super_fast2locals(self)
         # cellvars are values exported to inner scopes
@@ -146,7 +146,7 @@
                 w_name = self.space.wrap(name)
                 self.space.setitem(self.w_locals, w_name, w_value)
 
-    @jit.dont_look_inside
+    @jit.unroll_safe
     def locals2fast(self):
         super_locals2fast(self)
         freevarnames = self.pycode.co_cellvars + self.pycode.co_freevars
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -189,9 +189,8 @@
         # speed hack
         fresh_frame = jit.hint(frame, access_directly=True,
                                       fresh_virtualizable=True)
-        args_matched = args.parse_into_scope(None, fresh_frame.locals_stack_w,
-                                             func.name,
-                                             sig, func.defs_w)
+        args.parse_into_scope(None, fresh_frame.locals_stack_w, func.name,
+                              sig, func.defs_w)
         fresh_frame.init_cells()
         return frame.run()
 
@@ -202,9 +201,8 @@
         # speed hack
         fresh_frame = jit.hint(frame, access_directly=True,
                                       fresh_virtualizable=True)
-        args_matched = args.parse_into_scope(w_obj, fresh_frame.locals_stack_w,
-                                             func.name,
-                                             sig, func.defs_w)
+        args.parse_into_scope(w_obj, fresh_frame.locals_stack_w, func.name,
+                              sig, func.defs_w)
         fresh_frame.init_cells()
         return frame.run()
 
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -8,7 +8,7 @@
 from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.tool.pairtype import extendabletype
 
-from pypy.interpreter import eval, pycode, pytraceback
+from pypy.interpreter import pycode, pytraceback
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, operationerrfmt
@@ -443,12 +443,7 @@
     def getcode(self):
         return hint(self.pycode, promote=True)
 
-    @jit.dont_look_inside
-    def getfastscope(self):
-        "Get the fast locals as a list."
-        return self.locals_stack_w
-
-    @jit.dont_look_inside
+    @jit.look_inside_iff(lambda self, scope_w: jit.isvirtual(scope_w))
     def setfastscope(self, scope_w):
         """Initialize the fast locals from a list of values,
         where the order is according to self.pycode.signature()."""
@@ -475,15 +470,15 @@
         self.w_locals = w_locals
         self.locals2fast()
 
+    @jit.unroll_safe
     def fast2locals(self):
         # Copy values from the fastlocals to self.w_locals
         if self.w_locals is None:
             self.w_locals = self.space.newdict()
         varnames = self.getcode().getvarnames()
-        fastscope_w = self.getfastscope()
-        for i in range(min(len(varnames), self.getfastscopelength())):
+        for i in range(min(len(varnames), self.getcode().co_nlocals)):
             name = varnames[i]
-            w_value = fastscope_w[i]
+            w_value = self.locals_stack_w[i]
             w_name = self.space.wrap(name)
             if w_value is not None:
                 self.space.setitem(self.w_locals, w_name, w_value)
@@ -494,11 +489,12 @@
                     if not e.match(self.space, self.space.w_KeyError):
                         raise
 
+    @jit.unroll_safe
     def locals2fast(self):
         # Copy values from self.w_locals to the fastlocals
         assert self.w_locals is not None
         varnames = self.getcode().getvarnames()
-        numlocals = self.getfastscopelength()
+        numlocals = self.getcode().co_nlocals
 
         new_fastlocals_w = [None] * numlocals
 
@@ -519,9 +515,6 @@
         This is overridden in nestedscope.py"""
         pass
 
-    def getfastscopelength(self):
-        return self.pycode.co_nlocals
-
     def getclosure(self):
         return None
 
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -32,14 +32,20 @@
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
 
-def get_jitcell_at(next_instr, is_being_profiled, bytecode):
+def make_greenkey_dict_key(next_instr, is_being_profiled):
     # use only uints as keys in the jit_cells dict, rather than
     # a tuple (next_instr, is_being_profiled)
-    key = (next_instr << 1) | r_uint(intmask(is_being_profiled))
+    return (
+        (next_instr << 1) |
+        r_uint(intmask(is_being_profiled))
+    )
+
+def get_jitcell_at(next_instr, is_being_profiled, bytecode):
+    key = make_greenkey_dict_key(next_instr, is_being_profiled)
     return bytecode.jit_cells.get(key, None)
 
 def set_jitcell_at(newcell, next_instr, is_being_profiled, bytecode):
-    key = (next_instr << 1) | r_uint(intmask(is_being_profiled))
+    key = make_greenkey_dict_key(next_instr, is_being_profiled)
     bytecode.jit_cells[key] = newcell
 
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -67,7 +67,7 @@
         # LOAD_GLOBAL of OFFSET
         ops = entry_bridge.ops_by_id('cond', opcode='LOAD_GLOBAL')
         assert log.opnames(ops) == ["guard_value",
-                                    "getfield_gc", "guard_value",
+                                    "guard_value",
                                     "getfield_gc", "guard_value",
                                     "guard_not_invalidated"]
         ops = entry_bridge.ops_by_id('add', opcode='LOAD_GLOBAL')
@@ -81,7 +81,7 @@
             p39 = getfield_gc(p38, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
             i40 = force_token()
             p41 = getfield_gc(p38, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
-            guard_isnull(p41, descr=...)
+            guard_value(p41, ConstPtr(ptr42), descr=...)
             i42 = getfield_gc(p38, descr=<FieldU pypy.interpreter.executioncontext.ExecutionContext.inst_profilefunc .*>)
             i43 = int_is_zero(i42)
             guard_true(i43, descr=...)
@@ -424,33 +424,32 @@
             """, [])
         loop, = log.loops_by_id('call', is_entry_bridge=True)
         assert loop.match("""
-            guard_value(i6, 1, descr=...)
-            guard_nonnull_class(p8, ConstClass(W_IntObject), descr=...)
-            guard_value(i4, 0, descr=...)
-            guard_value(p3, ConstPtr(ptr14), descr=...)
-            i15 = getfield_gc_pure(p8, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
-            i17 = int_lt(i15, 5000)
-            guard_true(i17, descr=...)
-            guard_value(p18, ConstPtr(ptr19), descr=...)
-            p20 = getfield_gc(p18, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>)
-            guard_value(p20, ConstPtr(ptr21), descr=...)
+            guard_value(i4, 1, descr=...)
+            guard_nonnull_class(p12, ConstClass(W_IntObject), descr=...)
+            guard_value(i8, 0, descr=...)
+            guard_value(p2, ConstPtr(ptr21), descr=...)
+            i22 = getfield_gc_pure(p12, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
+            i24 = int_lt(i22, 5000)
+            guard_true(i24, descr=...)
+            guard_value(p7, ConstPtr(ptr25), descr=...)
+            p26 = getfield_gc(p7, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>)
+            guard_value(p26, ConstPtr(ptr27), descr=...)
             guard_not_invalidated(descr=...)
-            # most importantly, there is no getarrayitem_gc here
-            p23 = call(ConstClass(getexecutioncontext), descr=<Callr . EF=1>)
-            p24 = getfield_gc(p23, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
-            i25 = force_token()
-            p26 = getfield_gc(p23, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
-            guard_isnull(p26, descr=...)
-            i27 = getfield_gc(p23, descr=<FieldU pypy.interpreter.executioncontext.ExecutionContext.inst_profilefunc .*>)
-            i28 = int_is_zero(i27)
-            guard_true(i28, descr=...)
-            p30 = getfield_gc(ConstPtr(ptr29), descr=<FieldP pypy.interpreter.nestedscope.Cell.inst_w_value .*>)
-            guard_nonnull_class(p30, ConstClass(W_IntObject), descr=...)
-            i32 = getfield_gc_pure(p30, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
-            i33 = int_add_ovf(i15, i32)
+            p29 = call(ConstClass(getexecutioncontext), descr=<Callr 8 EF=1>)
+            p30 = getfield_gc(p29, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
+            p31 = force_token()
+            p32 = getfield_gc(p29, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)
+            guard_value(p32, ConstPtr(ptr33), descr=...)
+            i34 = getfield_gc(p29, descr=<FieldU pypy.interpreter.executioncontext.ExecutionContext.inst_profilefunc .*>)
+            i35 = int_is_zero(i34)
+            guard_true(i35, descr=...)
+            p37 = getfield_gc(ConstPtr(ptr36), descr=<FieldP pypy.interpreter.nestedscope.Cell.inst_w_value .*>)
+            guard_nonnull_class(p37, ConstClass(W_IntObject), descr=...)
+            i39 = getfield_gc_pure(p37, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
+            i40 = int_add_ovf(i22, i39)
             guard_no_overflow(descr=...)
+            guard_not_invalidated(descr=...)
             --TICK--
-            p39 = same_as(...) # Should be killed by backend
         """)
 
     def test_local_closure_is_virtual(self):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_min_max.py b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
--- a/pypy/module/pypyjit/test_pypy_c/test_min_max.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
@@ -1,8 +1,7 @@
-import py
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
+
 class TestMinMax(BaseTestPyPyC):
-
     def test_min_max(self):
         def main():
             i=0
@@ -24,7 +23,6 @@
             --TICK--
             jump(..., descr=...)
         """)
-        
 
     def test_silly_max(self):
         def main():
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -408,3 +408,20 @@
         log = self.run(main, [300])
         loop, = log.loops_by_id("long_op")
         assert len(loop.ops_by_id("long_op")) == 0
+
+    def test_settrace(self):
+        def main(n):
+            import sys
+            sys.settrace(lambda *args, **kwargs: None)
+
+            def f():
+                return 1
+
+            while n:
+                n -= f()
+
+        log = self.run(main, [300])
+        loops = log.loops_by_filename(self.filepath)
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(loops) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -60,27 +60,27 @@
         assert log.result == main(500)
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-        i55 = int_gt(i43, 0)
-        guard_true(i55, descr=...)
-        p56 = force_token()
-        setfield_gc(p0, p56, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
-        i57 = call_release_gil(..., i36, 1, descr=<Calli 4 ii EF=6>)
+        i56 = int_gt(i44, 0)
+        guard_true(i56, descr=...)
+        p57 = force_token()
+        setfield_gc(p0, p57, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 8>)
+        i58 = call_release_gil(..., i37, 1, descr=<Calli 4 ii EF=6>)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
-        i58 = int_is_true(i57)
-        guard_true(i58, descr=...)
-        i59 = int_sub(i43, 1)
+        i59 = int_is_true(i58)
+        guard_true(i59, descr=...)
+        i60 = int_sub(i44, 1)
         guard_not_invalidated(descr=...)
-        p61 = force_token()
-        setfield_gc(p0, p61, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
-        i62 = call_release_gil(..., i36, 0, descr=<Calli 4 ii EF=6>)
+        p62 = force_token()
+        setfield_gc(p0, p62, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 8>)
+        i63 = call_release_gil(..., i37, 0, descr=<Calli 4 ii EF=6>)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
-        i63 = int_is_true(i62)
-        guard_false(i63, descr=...)
-        p64 = force_token()
-        setfield_gc(p0, p64, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .*>)
-        call_release_gil(..., i36, descr=<Callv 0 i EF=6>)
+        i64 = int_is_true(i63)
+        guard_false(i64, descr=...)
+        p65 = force_token()
+        setfield_gc(p0, p65, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 8>)
+        call_release_gil(..., i37, descr=<Callv 0 i EF=6>)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
         guard_not_invalidated(descr=...)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -75,11 +75,15 @@
                 # and there is a signal pending: we force the ticker to
                 # -1, which should ensure perform() is called quickly.
 
+    def perform(self, executioncontext, frame):
+        self._poll_for_signals()
+
     @jit.dont_look_inside
-    def perform(self, executioncontext, frame):
+    def _poll_for_signals(self):
         # Poll for the next signal, if any
         n = self.pending_signal
-        if n < 0: n = pypysig_poll()
+        if n < 0:
+            n = pypysig_poll()
         while n >= 0:
             if self.space.threadlocals.signals_enabled():
                 # If we are in the main thread, report the signal now,
@@ -87,7 +91,8 @@
                 self.pending_signal = -1
                 report_signal(self.space, n)
                 n = self.pending_signal
-                if n < 0: n = pypysig_poll()
+                if n < 0:
+                    n = pypysig_poll()
             else:
                 # Otherwise, arrange for perform() to be called again
                 # after we switch to the main thread.
diff --git a/pypy/tool/pypyjit_demo.py b/pypy/tool/pypyjit_demo.py
--- a/pypy/tool/pypyjit_demo.py
+++ b/pypy/tool/pypyjit_demo.py
@@ -1,22 +1,8 @@
+def f():
+    i = 0
+    while i < 1303:
+        i += 1
+    return i
 
-def g(i):
-    k = 0
-    while k < 3:
-        k += 1
-    return i + 1
 
-def f(x):
-    for i in range(10000):
-        t = (1, 2, i)
-        i = g(i)
-        x == t
-
-
-
-try:
-    f((1, 2, 3))
-
-except Exception, e:
-    print "Exception: ", type(e)
-    print e
-
+f()

From noreply at buildbot.pypy.org  Sat Oct 26 00:36:47 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 00:36:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20131025223647.241A51C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67623:b2974d88f4d3
Date: 2013-10-25 15:36 -0700
http://bitbucket.org/pypy/pypy/changeset/b2974d88f4d3/

Log:	merged upstream

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -166,10 +166,9 @@
                 self._prepare_array_index(space, w_index)
         if iter_shape is None:
             # w_index is a list of slices
-            w_value = convert_to_array(space, w_value)
             chunks = self.implementation._prepare_slice_args(space, w_index)
             view = chunks.apply(space, self)
-            view.implementation.setslice(space, w_value)
+            view.implementation.setslice(space, val_arr)
             return
         loop.setitem_array_int(space, self, iter_shape, indexes, val_arr,
                                prefix)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -58,8 +58,6 @@
     specialize.argtype(1)(func)
     @functools.wraps(func)
     def dispatcher(self, v):
-        from pypy.module.micronumpy.interp_boxes import W_GenericBox
-        assert isinstance(v, W_GenericBox)
         return self.box_component(
             func(
                 self,

From noreply at buildbot.pypy.org  Sat Oct 26 00:50:54 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 26 Oct 2013 00:50:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20131025225054.22FE51C00D8@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67624:2e294c86125d
Date: 2013-10-25 11:52 -0700
http://bitbucket.org/pypy/pypy/changeset/2e294c86125d/

Log:	merge default

diff too long, truncating to 2000 out of 2763 lines

diff --git a/lib_pypy/numpypy/lib/utils.py b/lib_pypy/numpypy/lib/utils.py
--- a/lib_pypy/numpypy/lib/utils.py
+++ b/lib_pypy/numpypy/lib/utils.py
@@ -21,14 +21,4 @@
         ...
 
     """
-    try:
-        import numpy
-    except:
-        # running from pypy source directory
-        head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
-        return os.path.join(head, '../include')
-    else:
-        # using installed numpy core headers
-        import numpy.core as core
-        d = os.path.join(os.path.dirname(core.__file__), 'include')
-    return d
+    return os.path.join(os.path.dirname(__file__), '../../../include')
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -266,6 +266,7 @@
 You cannot use most existing standard library modules from RPython.  The
 exceptions are
 some functions in ``os``, ``math`` and ``time`` that have native support.
+We have our own "RPython standard library" in ``rpython.rlib.*``.
 
 To read more about the RPython limitations read the `RPython description`_.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -111,3 +111,6 @@
 
 .. branch: incremental-gc
 Added the new incminimark GC which performs GC in incremental steps
+
+.. branch: fast_cffi_list_init
+fastpath for cffi.new("long[]")
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -240,6 +240,18 @@
         # _____ this code is here to support testing only _____
         return self
 
+    def unpackiterable_int(self, space):
+        lst = space.listview_int(self)
+        if lst:
+            return lst[:]
+        return None
+
+    def unpackiterable_float(self, space):
+        lst = space.listview_float(self)
+        if lst:
+            return lst[:]
+        return None
+
 
 class W_InterpIterable(W_Root):
     def __init__(self, space, w_iterable):
@@ -847,6 +859,22 @@
         return self._unpackiterable_known_length_jitlook(w_iterator,
                                                          expected_length)
 
+
+    def unpackiterable_int(self, w_obj):
+        """
+        Return a RPython list of unwrapped ints out of w_obj. The list is
+        guaranteed to be acopy of the actual data contained in w_obj, so you
+        can freely modify it. It might return None if not supported.
+        """
+        return w_obj.unpackiterable_int(self)
+
+    def unpackiterable_float(self, w_obj):
+        """
+        Same as unpackiterable_int, but for floats.
+        """
+        return w_obj.unpackiterable_float(self)
+
+
     def length_hint(self, w_obj, default):
         """Return the length of an object, consulting its __length_hint__
         method if necessary.
@@ -904,6 +932,20 @@
         """
         return None
 
+    def listview_int(self, w_list):
+        """ Return a list of unwrapped int out of a list of int. If the
+        argument is not a list or does not contain only int, return None.
+        May return None anyway.
+        """
+        return None
+
+    def listview_float(self, w_list):
+        """ Return a list of unwrapped float out of a list of float. If the
+        argument is not a list or does not contain only float, return None.
+        May return None anyway.
+        """
+        return None
+
     def view_as_kwargs(self, w_dict):
         """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
         strings and one of wrapped values. otherwise return (None, None)
diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -131,8 +131,7 @@
         if len(cellvars) != ncellvars:
             raise OperationError(self.space.w_TypeError,
                                  self.space.wrap("bad cellvars"))
-        if self.cells is not None:
-            self.cells[:ncellvars] = cellvars
+        self.cells[:ncellvars] = cellvars
 
     @jit.dont_look_inside
     def fast2locals(self):
@@ -171,8 +170,6 @@
 
     @jit.unroll_safe
     def init_cells(self):
-        if self.cells is None:
-            return
         args_to_copy = self.pycode._args_as_cellvars
         for i in range(len(args_to_copy)):
             argnum = args_to_copy[i]
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -282,6 +282,20 @@
     def iter(self):
         return self.ctype.iter(self)
 
+    def unpackiterable_int(self, space):
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_int_items(self)
+        return None
+
+    def unpackiterable_float(self, space):
+        from pypy.module._cffi_backend import ctypearray
+        ctype = self.ctype
+        if isinstance(ctype, ctypearray.W_CTypeArray):
+            return ctype.ctitem.unpack_list_of_float_items(self)
+        return None
+
     @specialize.argtype(1)
     def write_raw_signed_data(self, source):
         misc.write_raw_signed_data(self._cdata, source, self.ctype.size)
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -43,6 +43,15 @@
     def is_unichar_ptr_or_array(self):
         return False
 
+    def unpack_list_of_int_items(self, cdata):
+        return None
+
+    def unpack_list_of_float_items(self, cdata):
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        return False
+
     def newp(self, w_init):
         space = self.space
         raise operationerrfmt(space.w_TypeError,
@@ -163,6 +172,9 @@
                               "cdata '%s' does not support iteration",
                               self.name)
 
+    def unpackiterable_int(self, cdata):
+        return None
+
     def get_vararg_type(self):
         return self
 
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -2,6 +2,7 @@
 Primitives.
 """
 
+import sys
 from pypy.interpreter.error import operationerrfmt
 
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
@@ -85,7 +86,6 @@
             return self.space.wrapbytes(s)
         return W_CType.string(self, cdataobj, maxlen)
 
-
 class W_CTypePrimitiveCharOrUniChar(W_CTypePrimitive):
     _attrs_ = []
     is_primitive_integer = True
@@ -170,6 +170,9 @@
             sh = self.size * 8
             self.vmin = r_uint(-1) << (sh - 1)
             self.vrangemax = (r_uint(1) << sh) - 1
+        else:
+            self.vmin = r_uint(0)
+            self.vrangemax = r_uint(-1)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -202,6 +205,35 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_signed_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.LONG):
+            from rpython.rlib.rrawarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.LONGP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            if self.size == rffi.sizeof(rffi.LONG): # fastest path
+                from rpython.rlib.rrawarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.LONGP, cdata)
+                copy_list_to_raw_array(int_list, cdata)
+            else:
+                overflowed = misc.pack_list_to_raw_array_bounds(
+                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                if overflowed != 0:
+                    self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
     _attrs_            = ['value_fits_long', 'value_fits_ulong', 'vrangemax']
@@ -214,6 +246,8 @@
         self.value_fits_ulong = self.size <= rffi.sizeof(lltype.Unsigned)
         if self.value_fits_long:
             self.vrangemax = self._compute_vrange_max()
+        else:
+            self.vrangemax = r_uint(sys.maxint)
 
     def _compute_vrange_max(self):
         sh = self.size * 8
@@ -253,6 +287,24 @@
     def write_raw_integer_data(self, w_cdata, value):
         w_cdata.write_raw_unsigned_data(value)
 
+    def unpack_list_of_int_items(self, w_cdata):
+        if self.value_fits_long:
+            res = [0] * w_cdata.get_array_length()
+            misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata,
+                                                     self.size)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        int_list = self.space.listview_int(w_ob)
+        if int_list is not None:
+            overflowed = misc.pack_list_to_raw_array_bounds(
+                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            if overflowed != 0:
+                self._overflow(self.space.wrap(overflowed))
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
     _attrs_ = []
@@ -313,6 +365,34 @@
         value = space.float_w(space.float(w_ob))
         misc.write_raw_float_data(cdata, value, self.size)
 
+    def unpack_list_of_float_items(self, w_cdata):
+        if self.size == rffi.sizeof(rffi.DOUBLE):
+            from rpython.rlib.rrawarray import populate_list_from_raw_array
+            res = []
+            buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata)
+            length = w_cdata.get_array_length()
+            populate_list_from_raw_array(res, buf, length)
+            return res
+        elif self.size == rffi.sizeof(rffi.FLOAT):
+            res = [0.0] * w_cdata.get_array_length()
+            misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata)
+            return res
+        return None
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
+                from rpython.rlib.rrawarray import copy_list_to_raw_array
+                cdata = rffi.cast(rffi.DOUBLEP, cdata)
+                copy_list_to_raw_array(float_list, cdata)
+                return True
+            elif self.size == rffi.sizeof(rffi.FLOAT):
+                misc.pack_float_list_to_raw_array(float_list, cdata,
+                                                  rffi.FLOAT, rffi.FLOATP)
+                return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+
 
 class W_CTypePrimitiveLongDouble(W_CTypePrimitiveFloat):
     _attrs_ = []
@@ -323,7 +403,6 @@
         return misc.longdouble2str(lvalue)
 
     def cast(self, w_ob):
-        space = self.space
         if (isinstance(w_ob, cdataobj.W_CData) and
                 isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)):
             w_cdata = self.convert_to_object(w_ob._cdata)
@@ -367,3 +446,15 @@
         else:
             value = space.float_w(space.float(w_ob))
             self._to_longdouble_and_write(value, cdata)
+
+    # Cannot have unpack_list_of_float_items() here:
+    # 'list(array-of-longdouble)' returns a list of cdata objects,
+    # not a list of floats.
+
+    def pack_list_of_items(self, cdata, w_ob):
+        float_list = self.space.listview_float(w_ob)
+        if float_list is not None:
+            misc.pack_float_list_to_raw_array(float_list, cdata,
+                                             rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
+            return True
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -60,19 +60,26 @@
             value = rffi.cast(rffi.CCHARP, value)
         return cdataobj.W_CData(space, value, self)
 
+    def _convert_array_from_listview(self, cdata, w_ob):
+        if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
+            return
+        #
+        space = self.space
+        lst_w = space.listview(w_ob)
+        if self.length >= 0 and len(lst_w) > self.length:
+            raise operationerrfmt(space.w_IndexError,
+                "too many initializers for '%s' (got %d)",
+                                  self.name, len(lst_w))
+        ctitem = self.ctitem
+        for i in range(len(lst_w)):
+            ctitem.convert_from_object(cdata, lst_w[i])
+            cdata = rffi.ptradd(cdata, ctitem.size)
+
     def convert_array_from_object(self, cdata, w_ob):
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
-            lst_w = space.listview(w_ob)
-            if self.length >= 0 and len(lst_w) > self.length:
-                raise operationerrfmt(space.w_IndexError,
-                    "too many initializers for '%s' (got %d)",
-                                      self.name, len(lst_w))
-            ctitem = self.ctitem
-            for i in range(len(lst_w)):
-                ctitem.convert_from_object(cdata, lst_w[i])
-                cdata = rffi.ptradd(cdata, ctitem.size)
+            self._convert_array_from_listview(cdata, w_ob)
         elif (self.can_cast_anything or
               (self.ctitem.is_primitive_integer and
                self.ctitem.size == rffi.sizeof(lltype.Char))):
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -319,3 +319,47 @@
             _raw_memclear_tp(TP, TPP, dest)
             return
     raise NotImplementedError("bad clear size")
+
+# ____________________________________________________________
+
+def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) - vmin > vrangemax:
+                    return x      # overflow
+                ptr[i] = rffi.cast(TP, x)
+            return 0
+    raise NotImplementedError("bad integer size")
+
+ at specialize.arg(2)
+def pack_float_list_to_raw_array(float_list, target, TP, TPP):
+    target = rffi.cast(TPP, target)
+    for i in range(len(float_list)):
+        x = float_list[i]
+        target[i] = rffi.cast(TP, x)
+
+def unpack_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_unsigned_list_from_raw_array(int_list, source, size):
+    for TP, TPP in _prim_unsigned_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, source)
+            for i in range(len(int_list)):
+                int_list[i] = rffi.cast(lltype.Signed, ptr[i])
+            return
+    raise NotImplementedError("bad integer size")
+
+def unpack_cfloat_list_from_raw_array(float_list, source):
+    ptr = rffi.cast(rffi.FLOATP, source)
+    for i in range(len(float_list)):
+        float_list[i] = rffi.cast(lltype.Float, ptr[i])
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -117,13 +117,17 @@
 
 SF_MSVC_BITFIELDS = 1
 SF_GCC_ARM_BITFIELDS = 2
+SF_GCC_BIG_ENDIAN = 4
 
 if sys.platform == 'win32':
     DEFAULT_SFLAGS = SF_MSVC_BITFIELDS
-elif rffi_platform.getdefined('__arm__', ''):
-    DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
 else:
-    DEFAULT_SFLAGS = 0
+    if rffi_platform.getdefined('__arm__', ''):
+        DEFAULT_SFLAGS = SF_GCC_ARM_BITFIELDS
+    else:
+        DEFAULT_SFLAGS = 0
+    if sys.byteorder == 'big':
+        DEFAULT_SFLAGS |= SF_GCC_BIG_ENDIAN
 
 @unwrap_spec(name=str)
 def new_struct_type(space, name):
@@ -325,6 +329,9 @@
                     prev_bitfield_free -= fbitsize
                     field_offset_bytes = boffset / 8 - ftype.size
 
+                if sflags & SF_GCC_BIG_ENDIAN:
+                    bitshift = 8 * ftype.size - fbitsize- bitshift
+
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
                                            bitshift, fbitsize)
                 fields_list.append(fld)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2949,8 +2949,6 @@
     _test_bitfield_details(flag=2)
 
 def test_bitfield_as_big_endian():
-    if '__pypy__' in sys.builtin_module_names:
-        py.test.skip("no big endian machine supported on pypy for now")
     _test_bitfield_details(flag=4)
 
 
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -0,0 +1,272 @@
+# side-effect: FORMAT_LONGDOUBLE must be built before the first test
+from pypy.module._cffi_backend import misc
+from pypy.module._cffi_backend.ctypeobj import W_CType
+
+
+class AppTest_fast_path_from_list(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def setup_method(self, meth):
+        def forbidden(*args):
+            assert False, 'The slow path is forbidden'
+        self._original = W_CType.pack_list_of_items.im_func
+        W_CType.pack_list_of_items = forbidden
+
+    def teardown_method(self, meth):
+        W_CType.pack_list_of_items = self._original
+
+    def test_fast_init_from_list(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, None)
+        buf = _cffi_backend.newp(LONG_ARRAY, [1, 2, 3])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == 3
+
+    def test_fast_init_from_list_float(self):
+        import _cffi_backend
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        P_DOUBLE = _cffi_backend.new_pointer_type(DOUBLE)
+        DOUBLE_ARRAY = _cffi_backend.new_array_type(P_DOUBLE, None)
+        buf = _cffi_backend.newp(DOUBLE_ARRAY, [1.1, 2.2, 3.3])
+        assert buf[0] == 1.1
+        assert buf[1] == 2.2
+        assert buf[2] == 3.3
+
+    def test_fast_init_short_from_list(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, None)
+        buf = _cffi_backend.newp(SHORT_ARRAY, [1, -2, 3])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [40000])
+        raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [-40000])
+
+    def test_fast_init_longlong_from_list(self):
+        if type(2 ** 50) is long:
+            large_int = 2 ** 30
+        else:
+            large_int = 2 ** 50
+        import _cffi_backend
+        LONGLONG = _cffi_backend.new_primitive_type('long long')
+        P_LONGLONG = _cffi_backend.new_pointer_type(LONGLONG)
+        LONGLONG_ARRAY = _cffi_backend.new_array_type(P_LONGLONG, None)
+        buf = _cffi_backend.newp(LONGLONG_ARRAY, [1, -2, 3, large_int])
+        assert buf[0] == 1
+        assert buf[1] == -2
+        assert buf[2] == 3
+        assert buf[3] == large_int
+
+    def test_fast_init_ushort_from_list(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, None)
+        buf = _cffi_backend.newp(USHORT_ARRAY, [1, 2, 40000])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == 40000
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [70000])
+        raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [-1])
+
+    def test_fast_init_ulong_from_list(self):
+        import sys
+        import _cffi_backend
+        ULONG = _cffi_backend.new_primitive_type('unsigned long')
+        P_ULONG = _cffi_backend.new_pointer_type(ULONG)
+        ULONG_ARRAY = _cffi_backend.new_array_type(P_ULONG, None)
+        buf = _cffi_backend.newp(ULONG_ARRAY, [1, 2, sys.maxint])
+        assert buf[0] == 1
+        assert buf[1] == 2
+        assert buf[2] == sys.maxint
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-1])
+        raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-sys.maxint])
+
+    def test_fast_init_cfloat_from_list(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, None)
+        buf = _cffi_backend.newp(FLOAT_ARRAY, [1.25, -3.5])
+        assert buf[0] == 1.25
+        assert buf[1] == -3.5
+
+    def test_fast_init_clongdouble_from_list(self):
+        import _cffi_backend
+        LONGDOUBLE = _cffi_backend.new_primitive_type('long double')
+        P_LONGDOUBLE = _cffi_backend.new_pointer_type(LONGDOUBLE)
+        LONGDOUBLE_ARRAY = _cffi_backend.new_array_type(P_LONGDOUBLE, None)
+        buf = _cffi_backend.newp(LONGDOUBLE_ARRAY, [1.25, -3.5])
+        assert float(buf[0]) == 1.25
+        assert float(buf[1]) == -3.5
+
+    def test_fast_init_bool_from_list(self):
+        import _cffi_backend
+        BOOL = _cffi_backend.new_primitive_type('_Bool')
+        P_BOOL = _cffi_backend.new_pointer_type(BOOL)
+        BOOL_ARRAY = _cffi_backend.new_array_type(P_BOOL, None)
+        buf = _cffi_backend.newp(BOOL_ARRAY, [1, 0])
+        assert buf[0] == 1
+        assert buf[1] == 0
+        assert type(buf[1]) is int
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [2])
+        raises(OverflowError, _cffi_backend.newp, BOOL_ARRAY, [-1])
+
+
+class AppTest_fast_path_bug(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def test_bug_not_list_or_tuple(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY_2 = _cffi_backend.new_array_type(P_LONG, 2)
+        P_LONG_ARRAY_2 = _cffi_backend.new_pointer_type(LONG_ARRAY_2)
+        LONG_ARRAY_ARRAY = _cffi_backend.new_array_type(P_LONG_ARRAY_2, None)
+        raises(TypeError, _cffi_backend.newp, LONG_ARRAY_ARRAY, [set([4, 5])])
+
+
+class AppTest_fast_path_to_list(object):
+    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+
+    def setup_method(self, meth):
+        from pypy.interpreter import gateway
+        from rpython.rlib import rrawarray
+        #
+        self.count = 0
+        def get_count(*args):
+            return self.space.wrap(self.count)
+        self.w_get_count = self.space.wrap(gateway.interp2app(get_count))
+        #
+        original = rrawarray.populate_list_from_raw_array
+        def populate_list_from_raw_array(*args):
+            self.count += 1
+            return original(*args)
+        self._original = original
+        rrawarray.populate_list_from_raw_array = populate_list_from_raw_array
+        #
+        original2 = misc.unpack_list_from_raw_array
+        def unpack_list_from_raw_array(*args):
+            self.count += 1
+            return original2(*args)
+        self._original2 = original2
+        misc.unpack_list_from_raw_array = unpack_list_from_raw_array
+        #
+        original3 = misc.unpack_cfloat_list_from_raw_array
+        def unpack_cfloat_list_from_raw_array(*args):
+            self.count += 1
+            return original3(*args)
+        self._original3 = original3
+        misc.unpack_cfloat_list_from_raw_array = (
+            unpack_cfloat_list_from_raw_array)
+        #
+        original4 = misc.unpack_unsigned_list_from_raw_array
+        def unpack_unsigned_list_from_raw_array(*args):
+            self.count += 1
+            return original4(*args)
+        self._original4 = original4
+        misc.unpack_unsigned_list_from_raw_array = (
+            unpack_unsigned_list_from_raw_array)
+        #
+        self.w_runappdirect = self.space.wrap(self.runappdirect)
+
+
+    def teardown_method(self, meth):
+        from rpython.rlib import rrawarray
+        rrawarray.populate_list_from_raw_array = self._original
+        misc.unpack_list_from_raw_array = self._original2
+        misc.unpack_cfloat_list_from_raw_array = self._original3
+        misc.unpack_unsigned_list_from_raw_array = self._original4
+
+    def test_list_int(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, 3)
+        buf = _cffi_backend.newp(LONG_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_TypeError_if_no_length(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        P_LONG = _cffi_backend.new_pointer_type(LONG)
+        LONG_ARRAY = _cffi_backend.new_array_type(P_LONG, 3)
+        buf = _cffi_backend.newp(LONG_ARRAY)
+        pbuf = _cffi_backend.cast(P_LONG, buf)
+        raises(TypeError, "list(pbuf)")
+
+    def test_bug(self):
+        import _cffi_backend
+        LONG = _cffi_backend.new_primitive_type('long')
+        five = _cffi_backend.cast(LONG, 5)
+        raises(TypeError, list, five)
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        five_and_a_half = _cffi_backend.cast(DOUBLE, 5.5)
+        raises(TypeError, list, five_and_a_half)
+
+    def test_list_float(self):
+        import _cffi_backend
+        DOUBLE = _cffi_backend.new_primitive_type('double')
+        P_DOUBLE = _cffi_backend.new_pointer_type(DOUBLE)
+        DOUBLE_ARRAY = _cffi_backend.new_array_type(P_DOUBLE, 3)
+        buf = _cffi_backend.newp(DOUBLE_ARRAY)
+        buf[0] = 1.1
+        buf[1] = 2.2
+        buf[2] = 3.3
+        lst = list(buf)
+        assert lst == [1.1, 2.2, 3.3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_short(self):
+        import _cffi_backend
+        SHORT = _cffi_backend.new_primitive_type('short')
+        P_SHORT = _cffi_backend.new_pointer_type(SHORT)
+        SHORT_ARRAY = _cffi_backend.new_array_type(P_SHORT, 3)
+        buf = _cffi_backend.newp(SHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 3
+        lst = list(buf)
+        assert lst == [1, 2, 3]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_ushort(self):
+        import _cffi_backend
+        USHORT = _cffi_backend.new_primitive_type('unsigned short')
+        P_USHORT = _cffi_backend.new_pointer_type(USHORT)
+        USHORT_ARRAY = _cffi_backend.new_array_type(P_USHORT, 3)
+        buf = _cffi_backend.newp(USHORT_ARRAY)
+        buf[0] = 1
+        buf[1] = 2
+        buf[2] = 50505
+        lst = list(buf)
+        assert lst == [1, 2, 50505]
+        if not self.runappdirect:
+            assert self.get_count() == 1
+
+    def test_list_cfloat(self):
+        import _cffi_backend
+        FLOAT = _cffi_backend.new_primitive_type('float')
+        P_FLOAT = _cffi_backend.new_pointer_type(FLOAT)
+        FLOAT_ARRAY = _cffi_backend.new_array_type(P_FLOAT, 3)
+        buf = _cffi_backend.newp(FLOAT_ARRAY)
+        buf[0] = 1.25
+        buf[1] = -2.5
+        buf[2] = 3.75
+        lst = list(buf)
+        assert lst == [1.25, -2.5, 3.75]
+        if not self.runappdirect:
+            assert self.get_count() == 1
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -8,7 +8,7 @@
 from rpython.rlib.rawstorage import raw_storage_getitem, raw_storage_setitem, \
         free_raw_storage, alloc_raw_storage
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import widen
 from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError
 from pypy.module.micronumpy.base import W_NDimArray
@@ -43,7 +43,7 @@
                                     + self.start + step * i)
                     v.append(_v)
             if comp_type == 'int':
-                v = intmask(v)
+                v = widen(v)
             elif comp_type == 'float':
                 v = float(v)
             elif comp_type == 'complex':
@@ -100,10 +100,15 @@
     if count < 2:
         def arg_lt(a, b):
             # Does numpy do <= ?
-            return a[0] < b[0]
+            return a[0] < b[0] or b[0] != b[0] and a[0] == a[0]
     else:
         def arg_lt(a, b):
             for i in range(count):
+                if b[0][i] != b[0][i] and a[0][i] == a[0][i]:
+                    return True
+                elif b[0][i] == b[0][i] and a[0][i] != a[0][i]:
+                    return False
+            for i in range(count):
                 if a[0][i] < b[0][i]:
                     return True
                 elif a[0][i] > b[0][i]:
@@ -200,7 +205,7 @@
                                     + self.start + step * i)
                     v.append(_v)
             if comp_type == 'int':
-                v = intmask(v)
+                v = widen(v)
             elif comp_type == 'float':
                 v = float(v)
             elif comp_type == 'complex':
@@ -318,7 +323,8 @@
 
 all_types = (types.all_float_types + types.all_complex_types +
              types.all_int_types)
-all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
+all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__ and
+                                     not issubclass(i[0], types.BaseFloat16)]
 all_types = unrolling_iterable(all_types)
 
 class ArgSortCache(object):
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -117,12 +117,12 @@
                 shape[i] += axis_size
         a_dt = arr.get_dtype()
         if dtype.is_record_type() and a_dt.is_record_type():
-            #Record types must match
+            # Record types must match
             for f in dtype.fields:
                 if f not in a_dt.fields or \
                              dtype.fields[f] != a_dt.fields[f]:
                     raise OperationError(space.w_TypeError,
-                               space.wrap("record type mismatch"))
+                               space.wrap("invalid type promotion"))
         elif dtype.is_record_type() or a_dt.is_record_type():
             raise OperationError(space.w_TypeError,
                         space.wrap("invalid type promotion"))
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -63,6 +63,7 @@
 
 class PrimitiveBox(Box):
     _mixin_ = True
+    _immutable_fields_ = ['value']
 
     def __init__(self, value):
         self.value = value
@@ -82,11 +83,11 @@
         ret = builder.build()
 
         lltype.free(value, flavor="raw")
-
         return ret
 
 class ComplexBox(Box):
     _mixin_ = True
+    _immutable_fields_ = ['real', 'imag']
 
     def __init__(self, real, imag=0.):
         self.real = real
@@ -111,11 +112,11 @@
         ret = builder.build()
 
         lltype.free(value, flavor="raw")
-
         return ret
 
+
 class W_GenericBox(W_Root):
-    _attrs_ = ()
+    _attrs_ = []
 
     def descr__new__(space, w_subtype, __args__):
         raise operationerrfmt(space.w_TypeError,
@@ -125,12 +126,21 @@
     def get_dtype(self, space):
         return self._get_dtype(space)
 
+    def item(self, space):
+        return self.get_dtype(space).itemtype.to_builtin_type(space, self)
+
     def descr_str(self, space):
         return space.wrap(self.get_dtype(space).itemtype.str_format(self))
 
     def descr_format(self, space, w_spec):
         return space.format(self.item(space), w_spec)
 
+    def descr_hash(self, space):
+        return space.hash(self.item(space))
+
+    def descr_index(self, space):
+        return space.index(self.item(space))
+
     def descr_int(self, space):
         box = self.convert_to(W_LongBox._get_dtype(space))
         assert isinstance(box, W_LongBox)
@@ -222,12 +232,6 @@
         w_remainder = self.descr_rmod(space, w_other)
         return space.newtuple([w_quotient, w_remainder])
 
-    def descr_hash(self, space):
-        return space.hash(self.item(space))
-
-    def item(self, space):
-        return self.get_dtype(space).itemtype.to_builtin_type(space, self)
-
     def descr_any(self, space):
         value = space.is_true(self)
         return self.get_dtype(space).box(value)
@@ -260,7 +264,7 @@
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
 class W_NumberBox(W_GenericBox):
-    _attrs_ = ()
+    pass
 
 class W_IntegerBox(W_NumberBox):
     def int_w(self, space):
@@ -309,10 +313,10 @@
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter('ulonglong')
 
 class W_InexactBox(W_NumberBox):
-    _attrs_ = ()
+    pass
 
 class W_FloatingBox(W_InexactBox):
-    _attrs_ = ()
+    pass
 
 class W_Float16Box(W_FloatingBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float16")
@@ -323,9 +327,43 @@
 class W_Float64Box(W_FloatingBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float64")
 
+class W_ComplexFloatingBox(W_InexactBox):
+    def descr_get_real(self, space):
+        dtype = self._COMPONENTS_BOX._get_dtype(space)
+        box = self.convert_real_to(dtype)
+        assert isinstance(box, self._COMPONENTS_BOX)
+        return space.wrap(box)
+
+    def descr_get_imag(self, space):
+        dtype = self._COMPONENTS_BOX._get_dtype(space)
+        box = self.convert_imag_to(dtype)
+        assert isinstance(box, self._COMPONENTS_BOX)
+        return space.wrap(box)
+
+class W_Complex64Box(ComplexBox, W_ComplexFloatingBox):
+    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex64")
+    _COMPONENTS_BOX = W_Float32Box
+
+class W_Complex128Box(ComplexBox, W_ComplexFloatingBox):
+    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex128")
+    _COMPONENTS_BOX = W_Float64Box
+
+if long_double_size == 8:
+    W_FloatLongBox = W_Float64Box
+    W_ComplexLongBox = W_Complex128Box
+
+elif long_double_size in (12, 16):
+    class W_FloatLongBox(W_FloatingBox, PrimitiveBox):
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % (long_double_size * 8))
+
+    class W_ComplexLongBox(ComplexBox, W_ComplexFloatingBox):
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % (long_double_size * 16))
+        _COMPONENTS_BOX = W_FloatLongBox
+
 class W_FlexibleBox(W_GenericBox):
-    _attrs_ = ['ofs', 'dtype', 'arr']
-    _immutable_fields_ = ['ofs']
+    _attrs_ = ['arr', 'ofs', 'dtype']
+    _immutable_fields_ = ['arr', 'ofs', 'dtype']
+
     def __init__(self, arr, ofs, dtype):
         self.arr = arr # we have to keep array alive
         self.ofs = ofs
@@ -334,11 +372,6 @@
     def get_dtype(self, space):
         return self.arr.dtype
 
- at unwrap_spec(self=W_GenericBox)
-def descr_index(space, self):
-    return space.index(self.item(space))
-
-
 class W_VoidBox(W_FlexibleBox):
     def descr_getitem(self, space, w_item):
         from pypy.module.micronumpy.types import VoidType
@@ -388,7 +421,6 @@
         # XXX assert dtype is str type
         return self
 
-
 class W_StringBox(W_CharacterBox):
     def descr__new__string_box(space, w_subtype, w_arg):
         from pypy.module.micronumpy.interp_dtype import new_string_dtype
@@ -398,7 +430,6 @@
             arr.storage[i] = arg[i]
         return W_StringBox(arr, 0, arr.dtype)
 
-
 class W_UnicodeBox(W_CharacterBox):
     def descr__new__unicode_box(space, w_subtype, w_arg):
         raise OperationError(space.w_NotImplementedError, space.wrap("Unicode is not supported yet"))
@@ -413,45 +444,6 @@
         #    arr.storage[i] = arg[i]
         return W_UnicodeBox(arr, 0, arr.dtype)
 
-
-class W_ComplexFloatingBox(W_InexactBox):
-    _attrs_ = ()
-    def descr_get_real(self, space):
-        dtype = self._COMPONENTS_BOX._get_dtype(space)
-        box = self.convert_real_to(dtype)
-        assert isinstance(box, self._COMPONENTS_BOX)
-        return space.wrap(box)
-
-    def descr_get_imag(self, space):
-        dtype = self._COMPONENTS_BOX._get_dtype(space)
-        box = self.convert_imag_to(dtype)
-        assert isinstance(box, self._COMPONENTS_BOX)
-        return space.wrap(box)
-
-
-class W_Complex64Box(ComplexBox, W_ComplexFloatingBox):
-    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex64")
-    _COMPONENTS_BOX = W_Float32Box
-
-
-class W_Complex128Box(ComplexBox, W_ComplexFloatingBox):
-    descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex128")
-    _COMPONENTS_BOX = W_Float64Box
-
-
-if long_double_size == 8:
-    W_FloatLongBox = W_Float64Box
-    W_ComplexLongBox = W_Complex128Box
-
-elif long_double_size in (12, 16):
-    class W_FloatLongBox(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % (long_double_size * 8))
-
-    class W_ComplexLongBox(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % (long_double_size * 16))
-        _COMPONENTS_BOX = W_FloatLongBox
-
-
 W_GenericBox.typedef = TypeDef("generic",
     __module__ = "numpypy",
 
@@ -521,7 +513,7 @@
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_BoolBox.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_BoolBox.descr_index),
     __reduce__ = interp2app(W_BoolBox.descr_reduce),
 )
 
@@ -544,49 +536,49 @@
 W_Int8Box.typedef = TypeDef("int8", W_SignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int8Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int8Box.descr_index),
     __reduce__ = interp2app(W_Int8Box.descr_reduce),
 )
 
 W_UInt8Box.typedef = TypeDef("uint8", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt8Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt8Box.descr_index),
     __reduce__ = interp2app(W_UInt8Box.descr_reduce),
 )
 
 W_Int16Box.typedef = TypeDef("int16", W_SignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int16Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int16Box.descr_index),
     __reduce__ = interp2app(W_Int16Box.descr_reduce),
 )
 
 W_UInt16Box.typedef = TypeDef("uint16", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt16Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt16Box.descr_index),
     __reduce__ = interp2app(W_UInt16Box.descr_reduce),
 )
 
 W_Int32Box.typedef = TypeDef("int32", (W_SignedIntegerBox.typedef,) + MIXIN_32,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int32Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int32Box.descr_index),
     __reduce__ = interp2app(W_Int32Box.descr_reduce),
 )
 
 W_UInt32Box.typedef = TypeDef("uint32", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt32Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt32Box.descr_index),
     __reduce__ = interp2app(W_UInt32Box.descr_reduce),
 )
 
 W_Int64Box.typedef = TypeDef("int64", (W_SignedIntegerBox.typedef,) + MIXIN_64,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int64Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_Int64Box.descr_index),
     __reduce__ = interp2app(W_Int64Box.descr_reduce),
 )
 
@@ -600,7 +592,7 @@
 W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_UInt64Box.descr__new__.im_func),
-    __index__ = interp2app(descr_index),
+    __index__ = interp2app(W_UInt64Box.descr_index),
     __reduce__ = interp2app(W_UInt64Box.descr_reduce),
 )
 
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -1,4 +1,3 @@
-
 import sys
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, operationerrfmt
@@ -11,6 +10,12 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib import jit
 
+if sys.byteorder == 'little':
+    byteorder_prefix = '<'
+    nonnative_byteorder_prefix = '>'
+else:
+    byteorder_prefix = '>'
+    nonnative_byteorder_prefix = '<'
 
 UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
@@ -44,12 +49,11 @@
     out = base.W_NDimArray.from_shape(space, shape, dtype)
     return out
 
-
 class W_Dtype(W_Root):
     _immutable_fields_ = ["itemtype", "num", "kind", "shape"]
 
     def __init__(self, itemtype, num, kind, name, char, w_box_type,
-                 alternate_constructors=[], aliases=[],
+                 alternate_constructors=[], aliases=[], float_type=None,
                  fields=None, fieldnames=None, native=True, shape=[], subdtype=None):
         self.itemtype = itemtype
         self.num = num
@@ -59,10 +63,10 @@
         self.w_box_type = w_box_type
         self.alternate_constructors = alternate_constructors
         self.aliases = aliases
+        self.float_type = float_type
         self.fields = fields
         self.fieldnames = fieldnames
         self.native = native
-        self.float_type = None
         self.shape = list(shape)
         self.subdtype = subdtype
         if not subdtype:
@@ -80,6 +84,7 @@
 
     def build_and_convert(self, space, box):
         return self.itemtype.build_and_convert(space, self, box)
+
     def coerce(self, space, w_item):
         return self.itemtype.coerce(space, self, w_item)
 
@@ -148,7 +153,11 @@
 
     def eq(self, space, w_other):
         w_other = space.call_function(space.gettypefor(W_Dtype), w_other)
-        return space.is_w(self, w_other)
+        if space.is_w(self, w_other):
+            return True
+        if isinstance(w_other, W_Dtype):
+            return space.eq_w(self.descr_reduce(space), w_other.descr_reduce(space))
+        return False
 
     def descr_eq(self, space, w_other):
         return space.wrap(self.eq(space, w_other))
@@ -223,7 +232,7 @@
         return self.kind == SIGNEDLTR
 
     def is_complex_type(self):
-        return False
+        return self.kind == COMPLEXLTR
 
     def is_float_type(self):
         return (self.kind == FLOATINGLTR or self.float_type is not None)
@@ -296,18 +305,6 @@
         fields = space.getitem(w_data, space.wrap(4))
         self.set_fields(space, fields)
 
-class W_ComplexDtype(W_Dtype):
-    def __init__(self, itemtype, num, kind, name, char, w_box_type,
-                 alternate_constructors=[], aliases=[],
-                 fields=None, fieldnames=None, native=True, float_type=None):
-        W_Dtype.__init__(self, itemtype, num, kind, name, char, w_box_type,
-                 alternate_constructors=alternate_constructors, aliases=aliases,
-                 fields=fields, fieldnames=fieldnames, native=native)
-        self.float_type = float_type
-
-    def is_complex_type(self):
-        return True
-
 def dtype_from_list(space, w_lst):
     lst_w = space.listview(w_lst)
     fields = {}
@@ -341,38 +338,6 @@
     raise OperationError(space.w_NotImplementedError, space.wrap(
         "dtype from dict"))
 
-def variable_dtype(space, name):
-    if name[0] in '<>=':
-        name = name[1:]
-    char = name[0]
-    if len(name) == 1:
-        size = 0
-    else:
-        try:
-            size = int(name[1:])
-        except ValueError:
-            raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
-    if char == 'S' or char == 'c':
-        itemtype = types.StringType(size)
-        basename = 'string'
-        num = 18
-        w_box_type = space.gettypefor(interp_boxes.W_StringBox)
-    elif char == 'V':
-        num = 20
-        basename = 'void'
-        itemtype = types.VoidType(size)
-        return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(size),
-                    "V", space.gettypefor(interp_boxes.W_VoidBox))
-    else:
-        assert char == 'U'
-        basename = 'unicode'
-        itemtype = types.UnicodeType(size)
-        num = 19
-        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
-    return W_Dtype(itemtype, num, char,
-                   basename + str(8 * itemtype.get_element_size()),
-                   char, w_box_type)
-
 def dtype_from_spec(space, name):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "dtype from spec"))
@@ -456,30 +421,61 @@
 )
 W_Dtype.typedef.acceptable_as_base_class = False
 
-if sys.byteorder == 'little':
-    byteorder_prefix = '<'
-    nonnative_byteorder_prefix = '>'
-else:
-    byteorder_prefix = '>'
-    nonnative_byteorder_prefix = '<'
+
+def variable_dtype(space, name):
+    if name[0] in '<>=':
+        name = name[1:]
+    char = name[0]
+    if len(name) == 1:
+        size = 0
+    else:
+        try:
+            size = int(name[1:])
+        except ValueError:
+            raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+    if char == 'c':
+        char = 'S'
+        size = 1
+    if char == 'S':
+        itemtype = types.StringType(size)
+        basename = 'string'
+        num = 18
+        w_box_type = space.gettypefor(interp_boxes.W_StringBox)
+    elif char == 'V':
+        num = 20
+        basename = 'void'
+        itemtype = types.VoidType(size)
+        return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(size),
+                    "V", space.gettypefor(interp_boxes.W_VoidBox))
+    else:
+        assert char == 'U'
+        basename = 'unicode'
+        itemtype = types.UnicodeType(size)
+        num = 19
+        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
+    return W_Dtype(itemtype, num, char,
+                   basename + str(8 * itemtype.get_element_size()),
+                   char, w_box_type)
 
 def new_string_dtype(space, size):
+    itemtype = types.StringType(size)
     return W_Dtype(
-        types.StringType(size),
+        itemtype,
         num=18,
         kind=STRINGLTR,
-        name='string',
-        char='S' + str(size),
+        name='string' + str(8 * itemtype.get_element_size()),
+        char='S',
         w_box_type = space.gettypefor(interp_boxes.W_StringBox),
     )
 
 def new_unicode_dtype(space, size):
+    itemtype = types.UnicodeType(size)
     return W_Dtype(
-        types.UnicodeType(size),
+        itemtype,
         num=19,
         kind=UNICODELTR,
-        name='unicode',
-        char='U' + str(size),
+        name='unicode' + str(8 * itemtype.get_element_size()),
+        char='U',
         w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
     )
 
@@ -613,7 +609,7 @@
             w_box_type=space.gettypefor(interp_boxes.W_FloatLongBox),
             aliases=["longdouble", "longfloat"],
         )
-        self.w_complex64dtype = W_ComplexDtype(
+        self.w_complex64dtype = W_Dtype(
             types.Complex64(),
             num=14,
             kind=COMPLEXLTR,
@@ -622,7 +618,7 @@
             w_box_type = space.gettypefor(interp_boxes.W_Complex64Box),
             float_type = self.w_float32dtype,
         )
-        self.w_complex128dtype = W_ComplexDtype(
+        self.w_complex128dtype = W_Dtype(
             types.Complex128(),
             num=15,
             kind=COMPLEXLTR,
@@ -633,7 +629,7 @@
             aliases=["complex"],
             float_type = self.w_float64dtype,
         )
-        self.w_complexlongdtype = W_ComplexDtype(
+        self.w_complexlongdtype = W_Dtype(
             types.ComplexLong(),
             num=16,
             kind=COMPLEXLTR,
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -18,10 +18,8 @@
 
 
 class W_Ufunc(W_Root):
-    _attrs_ = ["name", "promote_to_float", "promote_bools", "identity",
-               "allow_bool", "allow_complex", "complex_to_float"]
-    _immutable_fields_ = ["promote_to_float", "promote_bools", "name",
-            "allow_bool", "allow_complex", "complex_to_float"]
+    _immutable_fields_ = ["name", "promote_to_float", "promote_bools", "identity",
+            "int_only", "allow_bool", "allow_complex", "complex_to_float"]
 
     def __init__(self, name, promote_to_float, promote_bools, identity,
                  int_only, allow_bool, allow_complex, complex_to_float):
@@ -254,14 +252,12 @@
         return res
 
 class W_Ufunc1(W_Ufunc):
+    _immutable_fields_ = ["func", "bool_result"]
     argcount = 1
 
-    _immutable_fields_ = ["func", "name"]
-
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None, bool_result=False, int_only=False,
-        allow_bool=True, allow_complex=True, complex_to_float=False):
-
+            identity=None, bool_result=False, int_only=False,
+            allow_bool=True, allow_complex=True, complex_to_float=False):
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity,
                          int_only, allow_bool, allow_complex, complex_to_float)
         self.func = func
@@ -322,13 +318,12 @@
 
 
 class W_Ufunc2(W_Ufunc):
-    _immutable_fields_ = ["comparison_func", "func", "name", "int_only"]
+    _immutable_fields_ = ["func", "comparison_func", "done_func"]
     argcount = 2
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None, comparison_func=False, int_only=False,
-        allow_bool=True, allow_complex=True, complex_to_float=False):
-
+            identity=None, comparison_func=False, int_only=False,
+            allow_bool=True, allow_complex=True, complex_to_float=False):
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity,
                          int_only, allow_bool, allow_complex, complex_to_float)
         self.func = func
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -10,7 +10,7 @@
 from rpython.rlib.rcomplex import c_pow
 
 
-def rAlmostEqual(a, b, rel_err=2e-15, abs_err=5e-323, msg='', isnumpy=False):
+def rAlmostEqual(a, b, rel_err=2e-15, abs_err=5e-323, msg=''):
     """Fail if the two floating-point numbers are not almost equal.
 
     Determine whether floating-point values a and b are equal to within
@@ -36,7 +36,7 @@
     # (in theory there are examples where it would be legitimate for a
     # and b to have opposite signs; in practice these hardly ever
     # occur).
-    if not a and not b and not isnumpy:
+    if not a and not b:
         # only check it if we are running on top of CPython >= 2.6
         if sys.version_info >= (2, 6) and copysign(1., a) != copysign(1., b):
             raise AssertionError( msg + \
@@ -112,8 +112,6 @@
                     (k, space.unwrap(v))
                     for k, v in kwargs.iteritems()
                 ])
-                if '__pypy__' not in sys.builtin_module_names:
-                    kwargs['isnumpy'] = True
                 return space.wrap(rAlmostEqual(*args, **kwargs))
             cls.w_rAlmostEqual = cls.space.wrap(interp2app(cls_rAlmostEqual))
             def cls_c_pow(space, args_w):
@@ -616,9 +614,9 @@
         import numpypy as np
         rAlmostEqual = self.rAlmostEqual
 
-        for complex_, abs_err, testcases in (\
-                 (np.complex128, 5e-323, self.testcases128),
-                 # (np.complex64,  5e-32,  self.testcases64),
+        for complex_, testcases in (
+                 (np.complex128, self.testcases128),
+                 #(np.complex64, self.testcases64),
                 ):
             for id, fn, ar, ai, er, ei, flags in testcases:
                 arg = complex_(complex(ar, ai))
@@ -647,7 +645,7 @@
                 if fn in ('log', 'log10'):
                     real_abs_err = 2e-15
                 else:
-                    real_abs_err = abs_err
+                    real_abs_err = 5e-323
 
                 error_message = (
                     '%s: %s(%r(%r, %r))\n'
@@ -660,9 +658,9 @@
                 # since rAlmostEqual is a wrapped function,
                 # convert arguments to avoid boxed values
                 rAlmostEqual(float(expected[0]), float(actual[0]),
-                               abs_err=real_abs_err, msg=error_message)
+                             abs_err=real_abs_err, msg=error_message)
                 rAlmostEqual(float(expected[1]), float(actual[1]),
-                                   msg=error_message)
+                             msg=error_message)
                 sys.stderr.write('.')
             sys.stderr.write('\n')
 
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -702,7 +702,7 @@
         from numpypy import dtype
         nnp = self.non_native_prefix
         byteorder = self.native_prefix
-        assert dtype('i8') == dtype(byteorder + 'i8') == dtype('=i8') # XXX should be equal == dtype(long)
+        assert dtype('i8') == dtype(byteorder + 'i8') == dtype('=i8') == dtype(long)
         assert dtype(nnp + 'i8') != dtype('i8')
         assert dtype(nnp + 'i8').byteorder == nnp
         assert dtype('=i8').byteorder == '='
@@ -799,6 +799,14 @@
         assert d.type is str_
         assert d.name == "string64"
         assert d.num == 18
+        for i in [1, 2, 3]:
+            d = dtype('c%d' % i)
+            assert d.itemsize == 1
+            assert d.kind == 'S'
+            assert d.type is str_
+            assert d.name == 'string8'
+            assert d.num == 18
+            assert d.str == '|S1'
 
     def test_unicode_dtype(self):
         from numpypy import dtype, unicode_
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1295,9 +1295,7 @@
         assert a.max() == 5.7
         b = array([])
         raises(ValueError, "b.max()")
-
-        if 0: # XXX too pedantic
-            assert list(zeros((0, 2)).max(axis=1)) == []
+        assert list(zeros((0, 2)).max(axis=1)) == []
 
     def test_max_add(self):
         from numpypy import array
@@ -1310,9 +1308,7 @@
         assert a.min() == -3.0
         b = array([])
         raises(ValueError, "b.min()")
-
-        if 0: # XXX too pedantic
-            assert list(zeros((0, 2)).min(axis=1)) == []
+        assert list(zeros((0, 2)).min(axis=1)) == []
 
     def test_argmax(self):
         from numpypy import array
@@ -1639,7 +1635,7 @@
         exc = raises(TypeError, concatenate,
                             (zeros((2,), dtype=[('x', int), ('y', float)]),
                             (zeros((2,), dtype=[('x', float), ('y', float)]))))
-        assert str(exc.value).startswith('record type mismatch')
+        assert str(exc.value).startswith('invalid type promotion')
         exc = raises(TypeError, concatenate, ([1], zeros((2,),
                                             dtype=[('x', int), ('y', float)])))
         assert str(exc.value).startswith('invalid type promotion')
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -1,33 +1,26 @@
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-class AppTestSupport(BaseNumpyAppTest):
-    def setup_class(cls):
-        import struct
-        BaseNumpyAppTest.setup_class.im_func(cls)
-        cls.w_data = cls.space.wrap(struct.pack('dddd', 1, 2, 3, 4))
-        cls.w_fdata = cls.space.wrap(struct.pack('f', 2.3))
-        cls.w_float16val = cls.space.wrap('\x00E') # 5.0 in float16
-        cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
-        cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
-        cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
-
+class AppTestSorting(BaseNumpyAppTest):
     def test_argsort_dtypes(self):
         from numpypy import array, arange
         assert array(2.0).argsort() == 0
         nnp = self.non_native_prefix
         for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
-                        nnp + 'i2', complex]:
+                      nnp + 'i2', complex]:
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
+            exp = list(a)
+            exp = sorted(range(len(exp)), key=exp.__getitem__)
             c = a.copy()
             res = a.argsort()
-            assert (res == [2, 3, 5, 1, 0, 4, 7, 8, 6]).all(), \
+            assert (res == exp).all(), \
                 'a,res,dtype %r,%r,%r' % (a,res,dtype)
             assert (a == c).all() # not modified
-            a = arange(100)
+
+            a = arange(100, dtype=dtype)
             assert (a.argsort() == a).all()
         raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
 
-    def test_argsort_nd(self):
+    def test_argsort_ndim(self):
         from numpypy import array
         a = array([[4, 2], [1, 3]])
         assert (a.argsort() == [[1, 0], [0, 1]]).all()
@@ -63,19 +56,20 @@
     def test_sort_dtypes(self):
         from numpypy import array, arange
         for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
-                        'i2', complex]:
+                      'i2', complex]:
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
-            b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
+            b = sorted(list(a))
             c = a.copy()
             a.sort()
             assert (a == b).all(), \
                 'a,orig,dtype %r,%r,%r' % (a,c,dtype)
-        a = arange(100)
-        c = a.copy()
-        a.sort()
-        assert (a == c).all()
 
-    def test_sort_dtypesi_nonnative(self):
+            a = arange(100, dtype=dtype)
+            c = a.copy()
+            a.sort()
+            assert (a == c).all()
+
+    def test_sort_nonnative(self):
         from numpypy import array
         nnp = self.non_native_prefix
         for dtype in [ nnp + 'i2']:
@@ -104,6 +98,9 @@
         assert [isnan(bb) for bb in b] == [isnan(aa) for aa in a[::-1]]
         assert (b[:2] == a[::-1][:2]).all()
 
+        b = a.argsort()
+        assert (b == [2, 1, 0]).all()
+
         # check complex
         a = zeros(9, dtype=complex128)
         a.real += [nan, nan, nan, 1, 0, 1, 1, 0, 0]
@@ -113,6 +110,9 @@
         assert [isnan(bb) for bb in b] == [isnan(aa) for aa in a[::-1]]
         assert (b[:4] == a[::-1][:4]).all()
 
+        b = a.argsort()
+        assert (b == [8, 7, 6, 5, 4, 3, 2, 1, 0]).all()
+
         # all c scalar sorts use the same code with different types
         # so it suffices to run a quick check with one type. The number
         # of sorted items must be greater than ~50 to check the actual
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -315,16 +315,16 @@
         for i in range(4):
             assert b[i] == reference[i]
 
-        for dtype in ['int8', 'int16', 'int32', 'int64',
-                      'uint8', 'uint16', 'uint32', 'uint64']:
+        for dtype in 'bBhHiIlLqQ':
+            a = array([-2, -1, 0, 1, 2], dtype)
             reference = [0, -1, 0, 1, 0]
+            dtype = a.dtype.name
             if dtype[0] == 'u':
                 reference[1] = 0
             elif dtype == 'int32':
                     reference[2] = -2147483648
             elif dtype == 'int64':
                     reference[2] = -9223372036854775808
-            a = array([-2, -1, 0, 1, 2], dtype)
             b = reciprocal(a)
             assert (b == reference).all()
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -68,7 +68,6 @@
         )
     return dispatcher
 
-
 def raw_unary_op(func):
     specialize.argtype(1)(func)
     @functools.wraps(func)
@@ -116,8 +115,6 @@
     return dispatcher
 
 class BaseType(object):
-    _attrs_ = ()
-
     SortRepr = None # placeholders for sorting classes, overloaded in sort.py
     Sort = None
 
@@ -324,8 +321,6 @@
         raw_storage_setitem(storage, i + offset, value)
 
 class Bool(BaseType, Primitive):
-    _attrs_ = ()
-
     T = lltype.Bool
     BoxType = interp_boxes.W_BoolBox
     format_code = "?"
@@ -511,7 +506,7 @@
         ans = 0
         if raw == 0:
             # XXX good place to warn
-            if self.T is rffi.INT or self.T is rffi.LONG:
+            if self.T is rffi.INT or self.T is rffi.LONG or self.T is rffi.LONGLONG:
                 ans = most_neg_value_of(self.T)
         elif abs(raw) == 1:
             ans = raw
@@ -542,101 +537,75 @@
     _mixin_ = True
 
 class Int8(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.SIGNEDCHAR
     BoxType = interp_boxes.W_Int8Box
     format_code = "b"
+
 NonNativeInt8 = Int8
 
 class UInt8(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.UCHAR
     BoxType = interp_boxes.W_UInt8Box
     format_code = "B"
+
 NonNativeUInt8 = UInt8
 
 class Int16(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.SHORT
     BoxType = interp_boxes.W_Int16Box
     format_code = "h"
 
 class NonNativeInt16(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.SHORT
     BoxType = interp_boxes.W_Int16Box
     format_code = "h"
 
 class UInt16(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.USHORT
     BoxType = interp_boxes.W_UInt16Box
     format_code = "H"
 
 class NonNativeUInt16(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.USHORT
     BoxType = interp_boxes.W_UInt16Box
     format_code = "H"
 
 class Int32(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.INT
     BoxType = interp_boxes.W_Int32Box
     format_code = "i"
 
 class NonNativeInt32(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.INT
     BoxType = interp_boxes.W_Int32Box
     format_code = "i"
 
 class UInt32(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.UINT
     BoxType = interp_boxes.W_UInt32Box
     format_code = "I"
 
 class NonNativeUInt32(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.UINT
     BoxType = interp_boxes.W_UInt32Box
     format_code = "I"
 
 class Long(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.LONG
     BoxType = interp_boxes.W_LongBox
     format_code = "l"
 
 class NonNativeLong(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.LONG
     BoxType = interp_boxes.W_LongBox
     format_code = "l"
 
 class ULong(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.ULONG
     BoxType = interp_boxes.W_ULongBox
     format_code = "L"
 
 class NonNativeULong(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.ULONG
     BoxType = interp_boxes.W_ULongBox
     format_code = "L"
@@ -655,8 +624,6 @@
     return self.box(value)
 
 class Int64(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.LONGLONG
     BoxType = interp_boxes.W_Int64Box
     format_code = "q"
@@ -664,8 +631,6 @@
     _coerce = func_with_new_name(_int64_coerce, '_coerce')
 
 class NonNativeInt64(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.LONGLONG
     BoxType = interp_boxes.W_Int64Box
     format_code = "q"
@@ -686,8 +651,6 @@
     return self.box(value)
 
 class UInt64(BaseType, Integer):
-    _attrs_ = ()
-
     T = rffi.ULONGLONG
     BoxType = interp_boxes.W_UInt64Box
     format_code = "Q"
@@ -695,8 +658,6 @@
     _coerce = func_with_new_name(_uint64_coerce, '_coerce')
 
 class NonNativeUInt64(BaseType, NonNativeInteger):
-    _attrs_ = ()
-
     T = rffi.ULONGLONG
     BoxType = interp_boxes.W_UInt64Box
     format_code = "Q"
@@ -1038,16 +999,57 @@
         swapped_value = byteswap(rffi.cast(self.T, value))
         raw_storage_setitem(storage, i + offset, swapped_value)
 
+class BaseFloat16(Float):
+    _mixin_ = True
+
+    _STORAGE_T = rffi.USHORT
+    T = rffi.SHORT
+    BoxType = interp_boxes.W_Float16Box
+
+    @specialize.argtype(1)
+    def box(self, value):
+        return self.BoxType(rffi.cast(rffi.DOUBLE, value))
+
+    def runpack_str(self, s):
+        assert len(s) == 2
+        fval = unpack_float(s, native_is_bigendian)
+        return self.box(fval)
+
+    def default_fromstring(self, space):
+        return self.box(-1.0)
+
+    def byteswap(self, w_v):
+        value = self.unbox(w_v)
+        hbits = float_pack(value,2)
+        swapped = byteswap(rffi.cast(self._STORAGE_T, hbits))
+        return self.box(float_unpack(r_ulonglong(swapped), 2))
+
+class Float16(BaseType, BaseFloat16):
+    def _read(self, storage, i, offset):
+        hbits = raw_storage_getitem(self._STORAGE_T, storage, i + offset)
+        return float_unpack(r_ulonglong(hbits), 2)
+
+    def _write(self, storage, i, offset, value):
+        hbits = float_pack(value,2)
+        raw_storage_setitem(storage, i + offset,
+                rffi.cast(self._STORAGE_T, hbits))
+
+class NonNativeFloat16(BaseType, BaseFloat16):
+    def _read(self, storage, i, offset):
+        hbits = raw_storage_getitem(self._STORAGE_T, storage, i + offset)
+        return float_unpack(r_ulonglong(byteswap(hbits)), 2)
+
+    def _write(self, storage, i, offset, value):
+        hbits = float_pack(value,2)
+        raw_storage_setitem(storage, i + offset,
+                byteswap(rffi.cast(self._STORAGE_T, hbits)))
+
 class Float32(BaseType, Float):
-    _attrs_ = ()
-
     T = rffi.FLOAT
     BoxType = interp_boxes.W_Float32Box
     format_code = "f"
 
 class NonNativeFloat32(BaseType, NonNativeFloat):
-    _attrs_ = ()
-
     T = rffi.FLOAT
     BoxType = interp_boxes.W_Float32Box
     format_code = "f"
@@ -1061,22 +1063,17 @@
         return bool(v)
 
 class Float64(BaseType, Float):
-    _attrs_ = ()
-
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
     format_code = "d"
 
 class NonNativeFloat64(BaseType, NonNativeFloat):
-    _attrs_ = ()
-
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
     format_code = "d"
 
 class ComplexFloating(object):
     _mixin_ = True
-    _attrs_ = ()
 
     def _coerce(self, space, w_item):
         w_item = space.call_function(space.w_complex, w_item)
@@ -1169,22 +1166,22 @@
         real, imag = box.real, box.imag
         return real, imag
 
-    def store(self, arr, i, offset, box):
-        real, imag = self.unbox(box)
-        raw_storage_setitem(arr.storage, i+offset, real)
-        raw_storage_setitem(arr.storage,
-                i+offset+rffi.sizeof(self.T), imag)
-
     def _read(self, storage, i, offset):
         real = raw_storage_getitem(self.T, storage, i + offset)
-        imag = raw_storage_getitem(self.T, storage,
-                              i + offset + rffi.sizeof(self.T))
+        imag = raw_storage_getitem(self.T, storage, i + offset + rffi.sizeof(self.T))
         return real, imag
 
     def read(self, arr, i, offset, dtype=None):
         real, imag = self._read(arr.storage, i, offset)
         return self.box_complex(real, imag)
 
+    def _write(self, storage, i, offset, value):
+        raw_storage_setitem(storage, i + offset, value[0])
+        raw_storage_setitem(storage, i + offset + rffi.sizeof(self.T), value[1])
+
+    def store(self, arr, i, offset, box):
+        self._write(arr.storage, i, offset, self.unbox(box))
+
     @complex_binary_op
     def add(self, v1, v2):
         return rcomplex.c_add(v1, v2)
@@ -1621,10 +1618,7 @@
     def zeros_like(self, v):
         return 0, 0
 
-
 class Complex64(ComplexFloating, BaseType):
-    _attrs_ = ()
-
     T = rffi.FLOAT
     BoxType = interp_boxes.W_Complex64Box
     ComponentBoxType = interp_boxes.W_Float32Box
@@ -1632,8 +1626,6 @@
 NonNativeComplex64 = Complex64
 
 class Complex128(ComplexFloating, BaseType):
-    _attrs_ = ()
-
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Complex128Box
     ComponentBoxType = interp_boxes.W_Float64Box
@@ -1643,13 +1635,12 @@
 if interp_boxes.long_double_size == 8:
     FloatLong = Float64
     NonNativeFloatLong = NonNativeFloat64
+
     ComplexLong = Complex128
     NonNativeComplexLong = NonNativeComplex128
 
 elif interp_boxes.long_double_size in (12, 16):
     class FloatLong(BaseType, Float):
-        _attrs_ = ()
-
         T = rffi.LONGDOUBLE
         BoxType = interp_boxes.W_FloatLongBox
 
@@ -1667,15 +1658,12 @@
     NonNativeFloatLong = FloatLong
 
     class ComplexLong(ComplexFloating, BaseType):
-        _attrs_ = ()
-
         T = rffi.LONGDOUBLE
         BoxType = interp_boxes.W_ComplexLongBox
         ComponentBoxType = interp_boxes.W_FloatLongBox
 
     NonNativeComplexLong = ComplexLong
 
-
 class BaseStringType(object):
     _mixin_ = True
 
@@ -1883,7 +1871,6 @@
 NonNativeVoidType = VoidType
 
 class RecordType(BaseType):
-
     T = lltype.Char
 
     def __init__(self, offsets_and_fields, size):
@@ -1967,50 +1954,3 @@
                 all_complex_types.append((tp, 'complex'))

From noreply at buildbot.pypy.org  Sat Oct 26 00:50:55 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 26 Oct 2013 00:50:55 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20131025225055.5DD8E1C00D8@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67625:ecf25bee47a9
Date: 2013-10-25 11:53 -0700
http://bitbucket.org/pypy/pypy/changeset/ecf25bee47a9/

Log:	2to3

diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -267,7 +267,7 @@
                 ),
                 ("test_DescrFromType", "METH_O",
                 """
-                    Signed typenum = PyInt_AsLong(args);
+                    Signed typenum = PyLong_AsLong(args);
                     return _PyArray_DescrFromType(typenum);
                 """
                 ),

From noreply at buildbot.pypy.org  Sat Oct 26 00:50:56 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 26 Oct 2013 00:50:56 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: backout b40380d0b61e (kill __long__) for
	py3k
Message-ID: <20131025225056.A91171C00D8@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67626:6c7dc561b8f5
Date: 2013-10-25 15:49 -0700
http://bitbucket.org/pypy/pypy/changeset/6c7dc561b8f5/

Log:	backout b40380d0b61e (kill __long__) for py3k

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -146,11 +146,6 @@
         assert isinstance(box, W_LongBox)
         return space.wrap(box.value)
 
-    def descr_long(self, space):
-        box = self.convert_to(W_Int64Box._get_dtype(space))
-        assert isinstance(box, W_Int64Box)
-        return space.wrap(box.value)
-
     def descr_float(self, space):
         box = self.convert_to(W_Float64Box._get_dtype(space))
         assert isinstance(box, W_Float64Box)
@@ -453,7 +448,6 @@
     __repr__ = interp2app(W_GenericBox.descr_str),
     __format__ = interp2app(W_GenericBox.descr_format),
     __int__ = interp2app(W_GenericBox.descr_int),
-    __long__ = interp2app(W_GenericBox.descr_long),
     __float__ = interp2app(W_GenericBox.descr_float),
     __bool__ = interp2app(W_GenericBox.descr_nonzero),
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -895,15 +895,6 @@
             return space.int(self.descr_getitem(space, space.wrap(0)))
         raise OperationError(space.w_TypeError, space.wrap("only length-1 arrays can be converted to Python scalars"))
 
-    def descr_long(self, space):
-        shape = self.get_shape()
-        if len(shape) == 0:
-            assert isinstance(self.implementation, scalar.Scalar)
-            return space.long(space.wrap(self.implementation.get_scalar_value()))
-        if shape == [1]:
-            return space.int(self.descr_getitem(space, space.wrap(0)))
-        raise OperationError(space.w_TypeError, space.wrap("only length-1 arrays can be converted to Python scalars"))
-
     def descr_float(self, space):
         shape = self.get_shape()
         if len(shape) == 0:
@@ -1030,7 +1021,6 @@
     __repr__ = interp2app(W_NDimArray.descr_repr),
     __str__ = interp2app(W_NDimArray.descr_str),
     __int__ = interp2app(W_NDimArray.descr_int),
-    __long__ = interp2app(W_NDimArray.descr_long),
     __float__ = interp2app(W_NDimArray.descr_float),
 
     __pos__ = interp2app(W_NDimArray.descr_pos),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1914,15 +1914,6 @@
         b = array([1, 2, 3, 4])
         assert (a == b) == False
 
-    def test__long__(self):
-        from numpypy import array
-        assert long(array(1)) == 1
-        assert long(array([1])) == 1
-        assert isinstance(long(array([1])), long)
-        assert isinstance(long(array([1, 2][0])), long)
-        assert raises(TypeError, "long(array([1, 2]))")
-        assert long(array([1.5])) == 1
-
     def test__int__(self):
         from numpypy import array
         assert int(array(1)) == 1

From noreply at buildbot.pypy.org  Sat Oct 26 01:04:28 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 01:04:28 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Draft blog post on coverage.py
Message-ID: <20131025230428.9898E1C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5097:43914da11da0
Date: 2013-10-25 16:04 -0700
http://bitbucket.org/pypy/extradoc/changeset/43914da11da0/

Log:	Draft blog post on coverage.py

diff --git a/blog/draft/coverage.rst b/blog/draft/coverage.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/coverage.rst
@@ -0,0 +1,78 @@
+Making coverage.py faster under PyPy
+====================================
+
+If you've ever tried to run your programs with ``coverage.py`` under PyPy,
+you've probably experienced some incredible slowness. Take this simple
+program:
+
+.. source-code:: python
+
+    def f():
+        return 1
+
+
+    def main():
+        i = 10000000
+        while i:
+            i -= f()
+
+    main()
+
+Running ``time coverage.py run test.py`` five times, and looking at the best
+run, here's how PyPy 2.1 stacks up against CPython 2.7.5:
+
++-------------------------------------------------+
+| Python        | Time    | Normalized to CPython |
++---------------+---------+-----------------------+
+| CPython 2.7.5 | 3.879s  | 1.0x                  |
+| PyPy 2.1      | 53.330s | 13.7x slower          |
++---------------+---------+-----------------------+
+
+Totally ridiculous. I got turned onto this problem because on one of my
+projects CPython takes about 1.5 minutes to run our test suite on the build
+bot, but PyPy takes 8-10 minutes.
+
+So I sat down to address it. And the results:
+
++-------------------------------------------------+
+| Python        | Time    | Normalized to CPython |
++---------------+---------+-----------------------+
+| CPython 2.7.5 | 3.879s  | 1.0x                  |
+| PyPy 2.1      | 53.330s | 13.7x slower          |
+| PyPy head     | 1.433s  | 2.7x faster           |
++---------------+---------+-----------------------+
+
+Not bad.
+
+Technical details
+-----------------
+
+So how'd we do it? Previous, using ``sys.settrace()`` (which ``coverage.py``
+uses under the hood) disabled the JIT. Except it didn't just disable the JIT,
+it did it in a particularly insidious way, the JIT had no idea it was being
+disabled!
+
+Instead, every time PyPy discovered one of your functions was a hotspot, it
+would start tracing, to observe what the program was doing, and right when it
+was about to finish, ``coverage`` would run, and it would cause the JIT to
+abort. Tracing is a slow process, it makes up for it by generating fast machine
+code at the end, but tracing is still incredibly slow. But we never actually
+got to the "generate fast machine code" stage. Instead we'd pay all the cost of
+tracing, but then we'd abort, and reap none of the benefits.
+
+To fix this, we adjusted some of the heuristics in the JIT, to better show it
+how ``sys.settrace()`` works. Previous the JIT saw "here's an opaque function
+which gets the frame object, I wonder if it messes with the frame!" Now we let
+the JIT look inside the trace function, so it's able to see that
+``coverage.py`` isn't messing with the frame in any weird ways, it's just
+reading the line number and file path out of it.
+
+I asked several friends in the VM implementation and research field if they
+were aware of any other research into making VMs stay fast when debugging tools
+like ``coverage.py`` are running. No one I spoke to was aware of any, so I'm
+pleased to say that PyPy is quite possibly the first VM to work on optimizing
+code in debugging mode! This is possible because of our years spent investing
+in meta-tracing research.
+
+Happy testing,
+Alex

From noreply at buildbot.pypy.org  Sat Oct 26 01:06:20 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 01:06:20 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Fix up tables
Message-ID: <20131025230620.EA85C1C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5098:bb499fa28163
Date: 2013-10-25 16:06 -0700
http://bitbucket.org/pypy/extradoc/changeset/bb499fa28163/

Log:	Fix up tables

diff --git a/blog/draft/coverage.rst b/blog/draft/coverage.rst
--- a/blog/draft/coverage.rst
+++ b/blog/draft/coverage.rst
@@ -23,8 +23,9 @@
 
 +-------------------------------------------------+
 | Python        | Time    | Normalized to CPython |
++===============+=========+=======================+
+| CPython 2.7.5 | 3.879s  | 1.0x                  |
 +---------------+---------+-----------------------+
-| CPython 2.7.5 | 3.879s  | 1.0x                  |
 | PyPy 2.1      | 53.330s | 13.7x slower          |
 +---------------+---------+-----------------------+
 
@@ -36,9 +37,11 @@
 
 +-------------------------------------------------+
 | Python        | Time    | Normalized to CPython |
++===============+=========+=======================+
+| CPython 2.7.5 | 3.879s  | 1.0x                  |
 +---------------+---------+-----------------------+
-| CPython 2.7.5 | 3.879s  | 1.0x                  |
 | PyPy 2.1      | 53.330s | 13.7x slower          |
++---------------+---------+-----------------------+
 | PyPy head     | 1.433s  | 2.7x faster           |
 +---------------+---------+-----------------------+
 

From noreply at buildbot.pypy.org  Sat Oct 26 01:07:19 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 01:07:19 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: one more fix
Message-ID: <20131025230719.1C1B31C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5099:4b0af7c8e75c
Date: 2013-10-25 16:07 -0700
http://bitbucket.org/pypy/extradoc/changeset/4b0af7c8e75c/

Log:	one more fix

diff --git a/blog/draft/coverage.rst b/blog/draft/coverage.rst
--- a/blog/draft/coverage.rst
+++ b/blog/draft/coverage.rst
@@ -21,7 +21,7 @@
 Running ``time coverage.py run test.py`` five times, and looking at the best
 run, here's how PyPy 2.1 stacks up against CPython 2.7.5:
 
-+-------------------------------------------------+
++---------------+---------+-----------------------+
 | Python        | Time    | Normalized to CPython |
 +===============+=========+=======================+
 | CPython 2.7.5 | 3.879s  | 1.0x                  |
@@ -35,7 +35,7 @@
 
 So I sat down to address it. And the results:
 
-+-------------------------------------------------+
++---------------+---------+-----------------------+
 | Python        | Time    | Normalized to CPython |
 +===============+=========+=======================+
 | CPython 2.7.5 | 3.879s  | 1.0x                  |

From noreply at buildbot.pypy.org  Sat Oct 26 01:13:38 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 01:13:38 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Mention that I didn't reivew the
 literature exhaustively
Message-ID: <20131025231338.D86DD1C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5100:a0d525fd94e7
Date: 2013-10-25 16:13 -0700
http://bitbucket.org/pypy/extradoc/changeset/a0d525fd94e7/

Log:	Mention that I didn't reivew the literature exhaustively

diff --git a/blog/draft/coverage.rst b/blog/draft/coverage.rst
--- a/blog/draft/coverage.rst
+++ b/blog/draft/coverage.rst
@@ -72,10 +72,11 @@
 
 I asked several friends in the VM implementation and research field if they
 were aware of any other research into making VMs stay fast when debugging tools
-like ``coverage.py`` are running. No one I spoke to was aware of any, so I'm
-pleased to say that PyPy is quite possibly the first VM to work on optimizing
-code in debugging mode! This is possible because of our years spent investing
-in meta-tracing research.
+like ``coverage.py`` are running. No one I spoke to was aware of any (but I
+didn't do a particularly exhaustive review of the literature, I just tweeted at
+a few people), so I'm pleased to say that PyPy is quite possibly the first VM
+to work on optimizing code in debugging mode! This is possible because of our
+years spent investing in meta-tracing research.
 
 Happy testing,
 Alex

From noreply at buildbot.pypy.org  Sat Oct 26 01:19:08 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 01:19:08 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: fixed typos (via cyli)
Message-ID: <20131025231908.9E2ED1C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5101:dd6af335e817
Date: 2013-10-25 16:18 -0700
http://bitbucket.org/pypy/extradoc/changeset/dd6af335e817/

Log:	fixed typos (via cyli)

diff --git a/blog/draft/coverage.rst b/blog/draft/coverage.rst
--- a/blog/draft/coverage.rst
+++ b/blog/draft/coverage.rst
@@ -50,7 +50,7 @@
 Technical details
 -----------------
 
-So how'd we do it? Previous, using ``sys.settrace()`` (which ``coverage.py``
+So how'd we do it? Previously, using ``sys.settrace()`` (which ``coverage.py``
 uses under the hood) disabled the JIT. Except it didn't just disable the JIT,
 it did it in a particularly insidious way, the JIT had no idea it was being
 disabled!
@@ -64,7 +64,7 @@
 tracing, but then we'd abort, and reap none of the benefits.
 
 To fix this, we adjusted some of the heuristics in the JIT, to better show it
-how ``sys.settrace()`` works. Previous the JIT saw "here's an opaque function
+how ``sys.settrace()`` works. Previously the JIT saw "here's an opaque function
 which gets the frame object, I wonder if it messes with the frame!" Now we let
 the JIT look inside the trace function, so it's able to see that
 ``coverage.py`` isn't messing with the frame in any weird ways, it's just

From noreply at buildbot.pypy.org  Sat Oct 26 01:22:05 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 01:22:05 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: fancy dash
Message-ID: <20131025232205.E9BD11C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5102:e2ad4c53afac
Date: 2013-10-25 16:21 -0700
http://bitbucket.org/pypy/extradoc/changeset/e2ad4c53afac/

Log:	fancy dash

diff --git a/blog/draft/coverage.rst b/blog/draft/coverage.rst
--- a/blog/draft/coverage.rst
+++ b/blog/draft/coverage.rst
@@ -52,7 +52,7 @@
 
 So how'd we do it? Previously, using ``sys.settrace()`` (which ``coverage.py``
 uses under the hood) disabled the JIT. Except it didn't just disable the JIT,
-it did it in a particularly insidious way, the JIT had no idea it was being
+it did it in a particularly insidious way &#8212; the JIT had no idea it was being
 disabled!
 
 Instead, every time PyPy discovered one of your functions was a hotspot, it

From noreply at buildbot.pypy.org  Sat Oct 26 01:25:41 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 01:25:41 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: rewrite
Message-ID: <20131025232541.688CA1C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5103:b710505c06d1
Date: 2013-10-25 16:25 -0700
http://bitbucket.org/pypy/extradoc/changeset/b710505c06d1/

Log:	rewrite

diff --git a/blog/draft/coverage.rst b/blog/draft/coverage.rst
--- a/blog/draft/coverage.rst
+++ b/blog/draft/coverage.rst
@@ -55,13 +55,13 @@
 it did it in a particularly insidious way &#8212; the JIT had no idea it was being
 disabled!
 
-Instead, every time PyPy discovered one of your functions was a hotspot, it
-would start tracing, to observe what the program was doing, and right when it
-was about to finish, ``coverage`` would run, and it would cause the JIT to
-abort. Tracing is a slow process, it makes up for it by generating fast machine
-code at the end, but tracing is still incredibly slow. But we never actually
-got to the "generate fast machine code" stage. Instead we'd pay all the cost of
-tracing, but then we'd abort, and reap none of the benefits.
+Instead, every time PyPy discovered that one of your functions was a hotspot,
+it would start tracing to observe what the program was doing, and right when it
+was about to finish, ``coverage`` would run and cause the JIT to abort. Tracing
+is a slow process, it makes up for it by generating fast machine code at the
+end, but tracing is still incredibly slow. But we never actually got to the
+"generate fast machine code" stage. Instead we'd pay all the cost of tracing,
+but then we'd abort, and reap none of the benefits.
 
 To fix this, we adjusted some of the heuristics in the JIT, to better show it
 how ``sys.settrace()`` works. Previously the JIT saw "here's an opaque function

From noreply at buildbot.pypy.org  Sat Oct 26 01:42:25 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 01:42:25 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: final rephase
Message-ID: <20131025234225.581981C00D8@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5104:927371b8cf17
Date: 2013-10-25 16:42 -0700
http://bitbucket.org/pypy/extradoc/changeset/927371b8cf17/

Log:	final rephase

diff --git a/blog/draft/coverage.rst b/blog/draft/coverage.rst
--- a/blog/draft/coverage.rst
+++ b/blog/draft/coverage.rst
@@ -64,11 +64,12 @@
 but then we'd abort, and reap none of the benefits.
 
 To fix this, we adjusted some of the heuristics in the JIT, to better show it
-how ``sys.settrace()`` works. Previously the JIT saw "here's an opaque function
-which gets the frame object, I wonder if it messes with the frame!" Now we let
-the JIT look inside the trace function, so it's able to see that
-``coverage.py`` isn't messing with the frame in any weird ways, it's just
-reading the line number and file path out of it.
+how ``sys.settrace(<tracefunc>)`` works. Previously the JIT saw it as an opaque
+function which gets the frame object, and couldn't tell whether or not it
+messed with the frame object. Now we let the JIT look inside the
+``<tracefunc>`` function, so it's able to see that ``coverage.py`` isn't
+messing with the frame in any weird ways, it's just reading the line number and
+file path out of it.
 
 I asked several friends in the VM implementation and research field if they
 were aware of any other research into making VMs stay fast when debugging tools

From noreply at buildbot.pypy.org  Sat Oct 26 01:43:49 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 01:43:49 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: fix for pygments
Message-ID: <20131025234349.5CEF51C042B@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5105:5f88260857ab
Date: 2013-10-25 16:43 -0700
http://bitbucket.org/pypy/extradoc/changeset/5f88260857ab/

Log:	fix for pygments

diff --git a/blog/draft/coverage.rst b/blog/draft/coverage.rst
--- a/blog/draft/coverage.rst
+++ b/blog/draft/coverage.rst
@@ -5,7 +5,7 @@
 you've probably experienced some incredible slowness. Take this simple
 program:
 
-.. source-code:: python
+.. code-block:: python
 
     def f():
         return 1

From noreply at buildbot.pypy.org  Sat Oct 26 03:42:08 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 03:42:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: document this branch
Message-ID: <20131026014208.060001C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67627:2751bc1b671a
Date: 2013-10-25 18:41 -0700
http://bitbucket.org/pypy/pypy/changeset/2751bc1b671a/

Log:	document this branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -114,3 +114,6 @@
 
 .. branch: fast_cffi_list_init
 fastpath for cffi.new("long[]")
+
+.. branch: remove-eval-frame
+remove a pointless abstraction

From noreply at buildbot.pypy.org  Sat Oct 26 03:42:34 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 03:42:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: document this branch to
Message-ID: <20131026014234.7C1D11C019E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67628:06b2c9fd9c98
Date: 2013-10-25 18:42 -0700
http://bitbucket.org/pypy/pypy/changeset/06b2c9fd9c98/

Log:	document this branch to

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -117,3 +117,7 @@
 
 .. branch: remove-eval-frame
 remove a pointless abstraction
+
+.. branch: jit-settrace
+Allow the jit to continue running when sys.settrace() is active, necessary to
+make coverage.py fast

From noreply at buildbot.pypy.org  Sat Oct 26 09:10:30 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 26 Oct 2013 09:10:30 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #113: Test and fix: Values of
 anonymous enums are not exposed
Message-ID: <20131026071030.894651C02C7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1381:fae3707c5431
Date: 2013-10-26 09:10 +0200
http://bitbucket.org/cffi/cffi/changeset/fae3707c5431/

Log:	Issue #113: Test and fix: Values of anonymous enums are not exposed

diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -378,6 +378,12 @@
         self.baseinttype = baseinttype
         self.build_c_name_with_marker()
 
+    def force_the_name(self, forcename):
+        StructOrUnionOrEnum.force_the_name(self, forcename)
+        if self.forcename is None:
+            name = self.get_official_name()
+            self.forcename = '$' + name.replace(' ', '_')
+
     def check_not_partial(self):
         if self.partial and not self.partial_resolved:
             from . import ffiplatform
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -616,6 +616,21 @@
     s.x = 17
     assert s.x == 17
 
+def test_anonymous_enum():
+    ffi = FFI()
+    ffi.cdef("enum { EE1 }; enum { EE2, EE3 };")
+    lib = ffi.verify("enum { EE1 }; enum { EE2, EE3 };")
+    assert lib.EE1 == 0
+    assert lib.EE2 == 0
+    assert lib.EE3 == 1
+
+def test_nonfull_anonymous_enum():
+    ffi = FFI()
+    ffi.cdef("enum { EE1, ... }; enum { EE3, ... };")
+    lib = ffi.verify("enum { EE2, EE1 }; enum { EE3 };")
+    assert lib.EE1 == 1
+    assert lib.EE3 == 0
+
 def test_nonfull_enum_syntax2():
     ffi = FFI()
     ffi.cdef("enum ee { EE1, EE2=\t..., EE3 };")

From noreply at buildbot.pypy.org  Sat Oct 26 19:32:39 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 19:32:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merged nestedscope's frame manipulation
 code into the proper classes
Message-ID: <20131026173239.44BE11C02C7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67629:ef88c5a4a263
Date: 2013-10-26 10:32 -0700
http://bitbucket.org/pypy/pypy/changeset/ef88c5a4a263/

Log:	Merged nestedscope's frame manipulation code into the proper classes

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -326,9 +326,6 @@
         self.builtin_modules = {}
         self.reloading_modules = {}
 
-        # import extra modules for side-effects
-        import pypy.interpreter.nestedscope     # register *_DEREF bytecodes
-
         self.interned_strings = {}
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -1,8 +1,5 @@
-from rpython.rlib import jit
 from rpython.tool.uid import uid
 
-from pypy.interpreter import function, pycode, pyframe
-from pypy.interpreter.astcompiler import consts
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.mixedmodule import MixedModule
@@ -73,151 +70,3 @@
             return self.get()
         except ValueError:
             raise OperationError(space.w_ValueError, space.wrap("Cell is empty"))
-
-
-super_initialize_frame_scopes = pyframe.PyFrame.initialize_frame_scopes
-super_fast2locals = pyframe.PyFrame.fast2locals
-super_locals2fast = pyframe.PyFrame.locals2fast
-
-
-class __extend__(pyframe.PyFrame):
-    """This class enhances a standard frame with nested scope abilities,
-    i.e. handling of cell/free variables."""
-
-    # Cell Vars:
-    #     my local variables that are exposed to my inner functions
-    # Free Vars:
-    #     variables coming from a parent function in which i'm nested
-    # 'closure' is a list of Cell instances: the received free vars.
-
-    @jit.unroll_safe
-    def initialize_frame_scopes(self, outer_func, code):
-        super_initialize_frame_scopes(self, outer_func, code)
-        ncellvars = len(code.co_cellvars)
-        nfreevars = len(code.co_freevars)
-        if not nfreevars:
-            if not ncellvars:
-                self.cells = []
-                return            # no self.cells needed - fast path
-        elif outer_func is None:
-            space = self.space
-            raise OperationError(space.w_TypeError,
-                                 space.wrap("directly executed code object "
-                                            "may not contain free variables"))
-        if outer_func and outer_func.closure:
-            closure_size = len(outer_func.closure)
-        else:
-            closure_size = 0
-        if closure_size != nfreevars:
-            raise ValueError("code object received a closure with "
-                                 "an unexpected number of free variables")
-        self.cells = [None] * (ncellvars + nfreevars)
-        for i in range(ncellvars):
-            self.cells[i] = Cell()
-        for i in range(nfreevars):
-            self.cells[i + ncellvars] = outer_func.closure[i]
-
-    def _getcells(self):
-        return self.cells
-
-    def _setcellvars(self, cellvars):
-        ncellvars = len(self.pycode.co_cellvars)
-        if len(cellvars) != ncellvars:
-            raise OperationError(self.space.w_TypeError,
-                                 self.space.wrap("bad cellvars"))
-        self.cells[:ncellvars] = cellvars
-
-    @jit.unroll_safe
-    def fast2locals(self):
-        super_fast2locals(self)
-        # cellvars are values exported to inner scopes
-        # freevars are values coming from outer scopes
-        freevarnames = list(self.pycode.co_cellvars)
-        if self.pycode.co_flags & consts.CO_OPTIMIZED:
-            freevarnames.extend(self.pycode.co_freevars)
-        for i in range(len(freevarnames)):
-            name = freevarnames[i]
-            cell = self.cells[i]
-            try:
-                w_value = cell.get()
-            except ValueError:
-                pass
-            else:
-                w_name = self.space.wrap(name)
-                self.space.setitem(self.w_locals, w_name, w_value)
-
-    @jit.unroll_safe
-    def locals2fast(self):
-        super_locals2fast(self)
-        freevarnames = self.pycode.co_cellvars + self.pycode.co_freevars
-        for i in range(len(freevarnames)):
-            name = freevarnames[i]
-            cell = self.cells[i]
-            w_name = self.space.wrap(name)
-            try:
-                w_value = self.space.getitem(self.w_locals, w_name)
-            except OperationError, e:
-                if not e.match(self.space, self.space.w_KeyError):
-                    raise
-            else:
-                cell.set(w_value)
-
-    @jit.unroll_safe
-    def init_cells(self):
-        args_to_copy = self.pycode._args_as_cellvars
-        for i in range(len(args_to_copy)):
-            argnum = args_to_copy[i]
-            if argnum >= 0:
-                self.cells[i].set(self.locals_stack_w[argnum])
-
-    def getfreevarname(self, index):
-        freevarnames = self.pycode.co_cellvars + self.pycode.co_freevars
-        return freevarnames[index]
-
-    def iscellvar(self, index):
-        # is the variable given by index a cell or a free var?
-        return index < len(self.pycode.co_cellvars)
-
-    ### extra opcodes ###
-
-    def LOAD_CLOSURE(self, varindex, next_instr):
-        # nested scopes: access the cell object
-        cell = self.cells[varindex]
-        w_value = self.space.wrap(cell)
-        self.pushvalue(w_value)
-
-    def LOAD_DEREF(self, varindex, next_instr):
-        # nested scopes: access a variable through its cell object
-        cell = self.cells[varindex]
-        try:
-            w_value = cell.get()
-        except ValueError:
-            varname = self.getfreevarname(varindex)
-            if self.iscellvar(varindex):
-                message = "local variable '%s' referenced before assignment" % varname
-                w_exc_type = self.space.w_UnboundLocalError
-            else:
-                message = ("free variable '%s' referenced before assignment"
-                           " in enclosing scope" % varname)
-                w_exc_type = self.space.w_NameError
-            raise OperationError(w_exc_type, self.space.wrap(message))
-        else:
-            self.pushvalue(w_value)
-
-    def STORE_DEREF(self, varindex, next_instr):
-        # nested scopes: access a variable through its cell object
-        w_newvalue = self.popvalue()
-        cell = self.cells[varindex]
-        cell.set(w_newvalue)
-
-    @jit.unroll_safe
-    def MAKE_CLOSURE(self, numdefaults, next_instr):
-        w_codeobj = self.popvalue()
-        codeobj = self.space.interp_w(pycode.PyCode, w_codeobj)
-        w_freevarstuple = self.popvalue()
-        freevars = [self.space.interp_w(Cell, cell)
-                    for cell in self.space.fixedview(w_freevarstuple)]
-        defaultarguments = self.popvalues(numdefaults)
-        fn = function.Function(self.space, codeobj, self.w_globals,
-                               defaultarguments, freevars)
-        self.pushvalue(self.space.wrap(fn))
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -10,9 +10,11 @@
 
 from pypy.interpreter import pycode, pytraceback
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.astcompiler import consts
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.executioncontext import ExecutionContext
+from pypy.interpreter.nestedscope import Cell
 from pypy.tool import stdlib_opcode
 
 # Define some opcodes used
@@ -26,8 +28,6 @@
     """Represents a frame for a regular Python function
     that needs to be interpreted.
 
-    See also pyopcode.PyStandardFrame and nestedscope.PyNestedScopeFrame.
-
     Public fields:
      * 'space' is the object space this frame is running in
      * 'code' is the PyCode object this frame runs
@@ -35,6 +35,12 @@
      * 'w_globals' is the attached globals dictionary
      * 'builtin' is the attached built-in module
      * 'valuestack_w', 'blockstack', control the interpretation
+
+    Cell Vars:
+        my local variables that are exposed to my inner functions
+    Free Vars:
+        variables coming from a parent function in which i'm nested
+    'closure' is a list of Cell instances: the received free vars.
     """
 
     __metaclass__ = extendabletype
@@ -119,6 +125,7 @@
         else:
             return self.space.builtin
 
+    @jit.unroll_safe
     def initialize_frame_scopes(self, outer_func, code):
         # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
         # class bodies only have CO_NEWLOCALS.
@@ -126,13 +133,36 @@
         # CO_OPTIMIZED: no locals dict needed at all
         # NB: this method is overridden in nestedscope.py
         flags = code.co_flags
-        if flags & pycode.CO_OPTIMIZED:
-            return
-        if flags & pycode.CO_NEWLOCALS:
-            self.w_locals = self.space.newdict(module=True)
+        if not (flags & pycode.CO_OPTIMIZED):
+            if flags & pycode.CO_NEWLOCALS:
+                self.w_locals = self.space.newdict(module=True)
+            else:
+                assert self.w_globals is not None
+                self.w_locals = self.w_globals
+
+        ncellvars = len(code.co_cellvars)
+        nfreevars = len(code.co_freevars)
+        if not nfreevars:
+            if not ncellvars:
+                self.cells = []
+                return            # no self.cells needed - fast path
+        elif outer_func is None:
+            space = self.space
+            raise OperationError(space.w_TypeError,
+                                 space.wrap("directly executed code object "
+                                            "may not contain free variables"))
+        if outer_func and outer_func.closure:
+            closure_size = len(outer_func.closure)
         else:
-            assert self.w_globals is not None
-            self.w_locals = self.w_globals
+            closure_size = 0
+        if closure_size != nfreevars:
+            raise ValueError("code object received a closure with "
+                                 "an unexpected number of free variables")
+        self.cells = [None] * (ncellvars + nfreevars)
+        for i in range(ncellvars):
+            self.cells[i] = Cell()
+        for i in range(nfreevars):
+            self.cells[i + ncellvars] = outer_func.closure[i]
 
     def run(self):
         """Start this frame's execution."""
@@ -320,7 +350,7 @@
         w = space.wrap
         nt = space.newtuple
 
-        cells = self._getcells()
+        cells = self.cells
         if cells is None:
             w_cells = space.w_None
         else:
@@ -489,6 +519,23 @@
                     if not e.match(self.space, self.space.w_KeyError):
                         raise
 
+        # cellvars are values exported to inner scopes
+        # freevars are values coming from outer scopes
+        freevarnames = list(self.pycode.co_cellvars)
+        if self.pycode.co_flags & consts.CO_OPTIMIZED:
+            freevarnames.extend(self.pycode.co_freevars)
+        for i in range(len(freevarnames)):
+            name = freevarnames[i]
+            cell = self.cells[i]
+            try:
+                w_value = cell.get()
+            except ValueError:
+                pass
+            else:
+                w_name = self.space.wrap(name)
+                self.space.setitem(self.w_locals, w_name, w_value)
+
+
     @jit.unroll_safe
     def locals2fast(self):
         # Copy values from self.w_locals to the fastlocals
@@ -510,19 +557,39 @@
 
         self.setfastscope(new_fastlocals_w)
 
+        freevarnames = self.pycode.co_cellvars + self.pycode.co_freevars
+        for i in range(len(freevarnames)):
+            name = freevarnames[i]
+            cell = self.cells[i]
+            w_name = self.space.wrap(name)
+            try:
+                w_value = self.space.getitem(self.w_locals, w_name)
+            except OperationError, e:
+                if not e.match(self.space, self.space.w_KeyError):
+                    raise
+            else:
+                cell.set(w_value)
+
+    @jit.unroll_safe
     def init_cells(self):
-        """Initialize cellvars from self.locals_stack_w.
-        This is overridden in nestedscope.py"""
-        pass
+        """
+        Initialize cellvars from self.locals_stack_w.
+        """
+        args_to_copy = self.pycode._args_as_cellvars
+        for i in range(len(args_to_copy)):
+            argnum = args_to_copy[i]
+            if argnum >= 0:
+                self.cells[i].set(self.locals_stack_w[argnum])
 
     def getclosure(self):
         return None
 
-    def _getcells(self):
-        return None
-
     def _setcellvars(self, cellvars):
-        pass
+        ncellvars = len(self.pycode.co_cellvars)
+        if len(cellvars) != ncellvars:
+            raise OperationError(self.space.w_TypeError,
+                                 self.space.wrap("bad cellvars"))
+        self.cells[:ncellvars] = cellvars
 
     def fget_code(self, space):
         return space.wrap(self.getcode())
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -4,16 +4,19 @@
 The rest, dealing with variables in optimized ways, is in nestedscope.py.
 """
 
-import sys
-from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter import gateway, function, eval, pyframe, pytraceback
-from pypy.interpreter.pycode import PyCode, BytecodeCorruption
-from rpython.tool.sourcetools import func_with_new_name
+from rpython.rlib import jit, rstackovf
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import jit, rstackovf
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rlib.debug import check_nonneg
+from rpython.tool.sourcetools import func_with_new_name
+
+from pypy.interpreter import (
+    gateway, function, eval, pyframe, pytraceback, pycode
+)
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import OperationError, operationerrfmt
+from pypy.interpreter.nestedscope import Cell
+from pypy.interpreter.pycode import PyCode, BytecodeCorruption
 from pypy.tool.stdlib_opcode import bytecode_spec
 
 def unaryoperation(operationname):
@@ -501,6 +504,44 @@
         assert w_newvalue is not None
         self.locals_stack_w[varindex] = w_newvalue
 
+    def getfreevarname(self, index):
+        freevarnames = self.pycode.co_cellvars + self.pycode.co_freevars
+        return freevarnames[index]
+
+    def iscellvar(self, index):
+        # is the variable given by index a cell or a free var?
+        return index < len(self.pycode.co_cellvars)
+
+    def LOAD_DEREF(self, varindex, next_instr):
+        # nested scopes: access a variable through its cell object
+        cell = self.cells[varindex]
+        try:
+            w_value = cell.get()
+        except ValueError:
+            varname = self.getfreevarname(varindex)
+            if self.iscellvar(varindex):
+                message = "local variable '%s' referenced before assignment" % varname
+                w_exc_type = self.space.w_UnboundLocalError
+            else:
+                message = ("free variable '%s' referenced before assignment"
+                           " in enclosing scope" % varname)
+                w_exc_type = self.space.w_NameError
+            raise OperationError(w_exc_type, self.space.wrap(message))
+        else:
+            self.pushvalue(w_value)
+
+    def STORE_DEREF(self, varindex, next_instr):
+        # nested scopes: access a variable through its cell object
+        w_newvalue = self.popvalue()
+        cell = self.cells[varindex]
+        cell.set(w_newvalue)
+
+    def LOAD_CLOSURE(self, varindex, next_instr):
+        # nested scopes: access the cell object
+        cell = self.cells[varindex]
+        w_value = self.space.wrap(cell)
+        self.pushvalue(w_value)
+
     def POP_TOP(self, oparg, next_instr):
         self.popvalue()
 
@@ -1185,6 +1226,18 @@
                                defaultarguments)
         self.pushvalue(self.space.wrap(fn))
 
+    @jit.unroll_safe
+    def MAKE_CLOSURE(self, numdefaults, next_instr):
+        w_codeobj = self.popvalue()
+        codeobj = self.space.interp_w(pycode.PyCode, w_codeobj)
+        w_freevarstuple = self.popvalue()
+        freevars = [self.space.interp_w(Cell, cell)
+                    for cell in self.space.fixedview(w_freevarstuple)]
+        defaultarguments = self.popvalues(numdefaults)
+        fn = function.Function(self.space, codeobj, self.w_globals,
+                               defaultarguments, freevars)
+        self.pushvalue(self.space.wrap(fn))
+
     def BUILD_SLICE(self, numargs, next_instr):
         if numargs == 3:
             w_step = self.popvalue()

From noreply at buildbot.pypy.org  Sat Oct 26 19:36:25 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 19:36:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: random cleanup
Message-ID: <20131026173625.EF9F71C02C7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67630:bee44e32191e
Date: 2013-10-26 10:35 -0700
http://bitbucket.org/pypy/pypy/changeset/bee44e32191e/

Log:	random cleanup

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -5,9 +5,9 @@
 """
 
 from rpython.rlib import jit, rstackovf
+from rpython.rlib.debug import check_nonneg
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_uint, intmask
-from rpython.rlib.debug import check_nonneg
 from rpython.tool.sourcetools import func_with_new_name
 
 from pypy.interpreter import (
@@ -142,7 +142,6 @@
 
     @jit.unroll_safe
     def dispatch_bytecode(self, co_code, next_instr, ec):
-        space = self.space
         while True:
             self.last_instr = intmask(next_instr)
             if not jit.we_are_jitted():
@@ -735,7 +734,6 @@
         space = self.space
         if nbargs == 0:
             frame = self
-            ec = self.space.getexecutioncontext()
             while frame:
                 if frame.last_exception is not None:
                     operror = frame.last_exception

From noreply at buildbot.pypy.org  Sat Oct 26 19:55:39 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 19:55:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: Try out the new rtd theme for our docs
Message-ID: <20131026175539.C38D21C02C7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67631:45debf5514b3
Date: 2013-10-26 10:54 -0700
http://bitbucket.org/pypy/pypy/changeset/45debf5514b3/

Log:	Try out the new rtd theme for our docs

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -204,3 +204,6 @@
    u'fast, compliant alternative implementation of the Python language',
    u'The PyPy Project', 1)
 ]
+
+# Enable the new ReadTheDocs theme
+RTD_NEW_THEME = True

From noreply at buildbot.pypy.org  Sat Oct 26 19:58:02 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 19:58:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: Format these blocks nicer
Message-ID: <20131026175802.BC1A31C02C7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67632:0095cce3360c
Date: 2013-10-26 10:57 -0700
http://bitbucket.org/pypy/pypy/changeset/0095cce3360c/

Log:	Format these blocks nicer

diff --git a/pypy/doc/getting-started.rst b/pypy/doc/getting-started.rst
--- a/pypy/doc/getting-started.rst
+++ b/pypy/doc/getting-started.rst
@@ -51,7 +51,9 @@
 ---------------
 
 PyPy is ready to be executed as soon as you unpack the tarball or the zip
-file, with no need to install it in any specific location::
+file, with no need to install it in any specific location:
+
+.. code-block:: console
 
     $ tar xf pypy-2.1.tar.bz2
     $ ./pypy-2.1/bin/pypy
@@ -71,14 +73,12 @@
 If you want to install 3rd party libraries, the most convenient way is to
 install distribute_ and pip_:
 
+.. code-block:: console
+
     $ curl -O http://python-distribute.org/distribute_setup.py
-
     $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py
-
     $ ./pypy-2.1/bin/pypy distribute_setup.py
-
     $ ./pypy-2.1/bin/pypy get-pip.py
-
     $ ./pypy-2.1/bin/pip install pygments  # for example
 
 3rd party libraries will be installed in ``pypy-2.1/site-packages``, and

From noreply at buildbot.pypy.org  Sat Oct 26 20:00:05 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 20:00:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: Make it clear this is a root command,
	not a comment
Message-ID: <20131026180005.C62A91C02C7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67633:9003de95a6f6
Date: 2013-10-26 10:59 -0700
http://bitbucket.org/pypy/pypy/changeset/9003de95a6f6/

Log:	Make it clear this is a root command, not a comment

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -211,7 +211,9 @@
 "OSError: externmod.so: cannot restore segment prot after reloc: Permission
 denied." This is caused by a slight abuse of the C compiler during
 configuration, and can be disabled by running the following command with root
-privileges::
+privileges:
+
+.. code-block:: console
 
     # setenforce 0
 

From noreply at buildbot.pypy.org  Sat Oct 26 20:05:13 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 20:05:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: Use the right markup for these
Message-ID: <20131026180513.512271C02C7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67634:2e91ff33d818
Date: 2013-10-26 11:04 -0700
http://bitbucket.org/pypy/pypy/changeset/2e91ff33d818/

Log:	Use the right markup for these

diff --git a/pypy/doc/jit-hooks.rst b/pypy/doc/jit-hooks.rst
--- a/pypy/doc/jit-hooks.rst
+++ b/pypy/doc/jit-hooks.rst
@@ -1,11 +1,11 @@
 JIT hooks in PyPy
 =================
 
-There are several hooks in the `pypyjit` module that may help you with
+There are several hooks in the ``pypyjit`` module that may help you with
 understanding what's pypy's JIT doing while running your program. There
-are three functions related to that coming from the `pypyjit` module:
+are three functions related to that coming from the ``pypyjit`` module:
 
-* `set_optimize_hook(callable)`::
+.. function:: set_optimize_hook(callable)
 
     Set a compiling hook that will be called each time a loop is optimized,
     but before assembler compilation. This allows adding additional
@@ -17,7 +17,7 @@
     Result value will be the resulting list of operations, or None
 
 
-* `set_compile_hook(callable)`::
+.. function:: set_compile_hook(callable)
 
     Set a compiling hook that will be called each time a loop is compiled.
 
@@ -28,7 +28,7 @@
     inside the jit hook is itself jitted, it will get compiled, but the
     jit hook won't be called for that.
 
-* `set_abort_hook(hook)`::
+.. function:: set_abort_hook(hook)
 
     Set a hook (callable) that will be called each time there is tracing
     aborted due to some reason.

From noreply at buildbot.pypy.org  Sat Oct 26 20:07:23 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 20:07:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: more markup
Message-ID: <20131026180723.9BB471C02C7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67635:14397ed1ec03
Date: 2013-10-26 11:06 -0700
http://bitbucket.org/pypy/pypy/changeset/14397ed1ec03/

Log:	more markup

diff --git a/pypy/doc/jit-hooks.rst b/pypy/doc/jit-hooks.rst
--- a/pypy/doc/jit-hooks.rst
+++ b/pypy/doc/jit-hooks.rst
@@ -11,7 +11,7 @@
     but before assembler compilation. This allows adding additional
     optimizations on Python level.
 
-    The callable will be called with the pypyjit.JitLoopInfo object.
+    The callable will be called with the ``pypyjit.JitLoopInfo`` object.
     Refer to it's documentation for details.
 
     Result value will be the resulting list of operations, or None
@@ -21,7 +21,7 @@
 
     Set a compiling hook that will be called each time a loop is compiled.
 
-    The callable will be called with the pypyjit.JitLoopInfo object.
+    The callable will be called with the ``pypyjit.JitLoopInfo`` object.
     Refer to it's documentation for details.
 
     Note that jit hook is not reentrant. It means that if the code
@@ -33,7 +33,8 @@
     Set a hook (callable) that will be called each time there is tracing
     aborted due to some reason.
 
-    The hook will be called as in: hook(jitdriver_name, greenkey, reason)
+    The hook will be invoked with the siagnture:
+    ``hook(jitdriver_name, greenkey, reason)``
 
     Reason is a string, the meaning of other arguments is the same
     as attributes on JitLoopInfo object

From noreply at buildbot.pypy.org  Sat Oct 26 20:08:35 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 20:08:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: Yet more markup
Message-ID: <20131026180835.D6D671C02C7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67636:96bed2d09b62
Date: 2013-10-26 11:07 -0700
http://bitbucket.org/pypy/pypy/changeset/96bed2d09b62/

Log:	Yet more markup

diff --git a/pypy/doc/jit/virtualizable.rst b/pypy/doc/jit/virtualizable.rst
--- a/pypy/doc/jit/virtualizable.rst
+++ b/pypy/doc/jit/virtualizable.rst
@@ -25,13 +25,19 @@
 forcing them. This is very useful for frames. Declaring an object to be
 virtualizable works like this:
 
+.. code-block:: python
+
     class Frame(object):
        _virtualizable_ = ['locals[*]', 'stackdepth']
 
-And we use them in ``JitDriver`` like this::
+
+And we use them in ``JitDriver`` like this:
+
+.. code-block:: python
 
     jitdriver = JitDriver(greens=[], reds=['frame'], virtualizables=['frame'])
 
+
 This declaration means that ``stackdepth`` is a virtualizable **field**, while
 ``locals`` is a virtualizable **array** (a list stored on a virtualizable).
 There are various rules about using virtualizables, especially using
@@ -44,9 +50,11 @@
   constant or changing rarely within the context of the user's code.
 
 * If you initialize a new virtualizable in the JIT, it has to be done like this
-  (for example if we're in ``Frame.__init__``)::
+  (for example if we're in ``Frame.__init__``):
 
-    self = hint(self, access_directly=True, fresh_virtualizable=True)
+  .. code-block:: python
+
+      self = hint(self, access_directly=True, fresh_virtualizable=True)
 
   that way you can populate the fields directly.
 

From noreply at buildbot.pypy.org  Sat Oct 26 20:58:02 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 26 Oct 2013 20:58:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: Bah,
 jumping ahead by a too-small amount means it's likely that
Message-ID: <20131026185802.162C11C00EC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67637:409b1a1452a6
Date: 2013-10-26 20:57 +0200
http://bitbucket.org/pypy/pypy/changeset/409b1a1452a6/

Log:	Bah, jumping ahead by a too-small amount means it's likely that two
	successive pieces will run largely identical tests.

diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -17,6 +17,6 @@
     cpu = CPU(None, None)
     cpu.setup_once()
     r = Random()
-    r.jumpahead(piece*per_piece)
+    r.jumpahead(piece*99999999)
     for i in range(piece*per_piece, (piece+1)*per_piece):
         check_random_function(cpu, LLtypeOperationBuilder, r, i, total_iterations)

From noreply at buildbot.pypy.org  Sat Oct 26 21:33:29 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 21:33:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix for 32bit
Message-ID: <20131026193329.ED8511C1054@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67638:afbf16b2b4ca
Date: 2013-10-26 12:30 -0700
http://bitbucket.org/pypy/pypy/changeset/afbf16b2b4ca/

Log:	fix for 32bit

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -435,7 +435,7 @@
             p26 = getfield_gc(p7, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>)
             guard_value(p26, ConstPtr(ptr27), descr=...)
             guard_not_invalidated(descr=...)
-            p29 = call(ConstClass(getexecutioncontext), descr=<Callr 8 EF=1>)
+            p29 = call(ConstClass(getexecutioncontext), descr=<Callr . EF=1>)
             p30 = getfield_gc(p29, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
             p31 = force_token()
             p32 = getfield_gc(p29, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_w_tracefunc .*>)

From noreply at buildbot.pypy.org  Sat Oct 26 21:33:31 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 26 Oct 2013 21:33:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20131026193331.3F8301C1405@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67639:205ceffda4fa
Date: 2013-10-26 12:32 -0700
http://bitbucket.org/pypy/pypy/changeset/205ceffda4fa/

Log:	merged upstream

diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -17,6 +17,6 @@
     cpu = CPU(None, None)
     cpu.setup_once()
     r = Random()
-    r.jumpahead(piece*per_piece)
+    r.jumpahead(piece*99999999)
     for i in range(piece*per_piece, (piece+1)*per_piece):
         check_random_function(cpu, LLtypeOperationBuilder, r, i, total_iterations)

From noreply at buildbot.pypy.org  Sun Oct 27 01:23:44 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 27 Oct 2013 01:23:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: random cleanups
Message-ID: <20131026232344.6BFB61C0050@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67640:061674dab643
Date: 2013-10-26 16:23 -0700
http://bitbucket.org/pypy/pypy/changeset/061674dab643/

Log:	random cleanups

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -412,7 +412,7 @@
                     done = True
                     break
         if block.next_block and not done:
-            max_depth = self._next_stack_depth_walk(block.next_block, depth)
+            self._next_stack_depth_walk(block.next_block, depth)
 
     def _build_lnotab(self, blocks):
         """Build the line number table for tracebacks and tracing."""
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -285,7 +285,6 @@
                         exec_node.column)
 
     def handle_assert_stmt(self, assert_node):
-        child_count = len(assert_node.children)
         expr = self.handle_expr(assert_node.children[1])
         msg = None
         if len(assert_node.children) == 4:
@@ -1061,7 +1060,6 @@
         if negative:
             raw = "-" + raw
         w_num_str = self.space.wrap(raw)
-        w_index = None
         w_base = self.space.wrap(base)
         if raw[-1] in "lL":
             tp = self.space.w_long
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -251,7 +251,6 @@
     def exec_host_bytecode(self, w_globals, w_locals):
         if sys.version_info < (2, 7):
             raise Exception("PyPy no longer supports Python 2.6 or lower")
-        from pypy.interpreter.pyframe import PyFrame
         frame = self.space.FrameClass(self.space, self, w_globals, None)
         frame.setdictscope(w_locals)
         return frame.run()

From noreply at buildbot.pypy.org  Sun Oct 27 04:21:13 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 27 Oct 2013 04:21:13 +0100 (CET)
Subject: [pypy-commit] pypy default: kill trailing whitespace and a few
	unused vars
Message-ID: <20131027032113.BA89E1C02C7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67641:181f9933574a
Date: 2013-10-26 20:20 -0700
http://bitbucket.org/pypy/pypy/changeset/181f9933574a/

Log:	kill trailing whitespace and a few unused vars

diff --git a/pypy/config/makerestdoc.py b/pypy/config/makerestdoc.py
--- a/pypy/config/makerestdoc.py
+++ b/pypy/config/makerestdoc.py
@@ -1,6 +1,6 @@
 import py
 from pypy.tool.rest.rst import Rest, Paragraph, Strong, ListItem, Title, Link
-from pypy.tool.rest.rst import Directive, Em, Quote, Text
+from pypy.tool.rest.rst import Directive, Text
 
 from rpython.config.config import ChoiceOption, BoolOption, StrOption, IntOption
 from rpython.config.config import FloatOption, OptionDescription, Option, Config
@@ -45,7 +45,7 @@
             content.add(ListItem(Strong("default:"), str(self.default)))
 
         requirements = []
-        
+
         for val in self.values:
             if val not in self._requires:
                 continue
@@ -138,9 +138,6 @@
         content.join(
             ListItem(Strong("name:"), self._name),
             ListItem(Strong("description:"), self.doc))
-        stack = []
-        curr = content
-        config = Config(self)
         return content
 
 
@@ -166,7 +163,6 @@
     for path in config.getpaths(include_groups=False):
         subconf, step = config._cfgimpl_get_home_by_path(path)
         fullpath = (descr._name + "." + path)
-        prefix = fullpath.rsplit(".", 1)[0]
         subdescr = getattr(subconf._cfgimpl_descr, step)
         cmdline = get_cmdline(subdescr.cmdline, fullpath)
         if cmdline is not None:
@@ -183,7 +179,7 @@
         curr.add(ListItem(Link(cmdline + ":", fullpath + ".html"),
                           Text(subdescr.doc)))
     return content
-    
+
 
 def register_config_role(docdir):
     """ register a :config: ReST link role for use in documentation. """
diff --git a/pypy/tool/rest/rst.py b/pypy/tool/rest/rst.py
--- a/pypy/tool/rest/rst.py
+++ b/pypy/tool/rest/rst.py
@@ -38,7 +38,7 @@
             class_list = parent_cls
         if obj.allow_nesting:
             class_list.append(obj)
-        
+
         for _class in class_list:
             if not _class.allowed_child:
                 _class.allowed_child = {obj:True}
@@ -56,7 +56,7 @@
     allow_nesting = False
     allowed_child = {}
     defaults = {}
-    
+
     _reg_whitespace = py.std.re.compile('\s+')
 
     def __init__(self, *args, **kwargs):
@@ -66,53 +66,53 @@
             self._add(child)
         for arg in kwargs:
             setattr(self, arg, kwargs[arg])
-    
+
     def join(self, *children):
         """ add child nodes
-        
+
             returns a reference to self
         """
         for child in children:
             self._add(child)
         return self
-    
+
     def add(self, child):
         """ adds a child node
-            
+
             returns a reference to the child
         """
         self._add(child)
         return child
-        
+
     def _add(self, child):
         if child.__class__ not in self.allowed_child:
             raise RestError("%r cannot be child of %r" % \
                 (child.__class__, self.__class__))
         self.children.append(child)
         child.parent = self
-    
+
     def __getitem__(self, item):
         return self.children[item]
-    
+
     def __setitem__(self, item, value):
         self.children[item] = value
 
     def text(self):
         """ return a ReST string representation of the node """
         return self.sep.join([child.text() for child in self.children])
-    
+
     def wordlist(self):
-        """ return a list of ReST strings for this node and its children """ 
+        """ return a list of ReST strings for this node and its children """
         return [self.text()]
 
 class Rest(AbstractNode):
     """ Root node of a document """
-    
+
     sep = "\n\n"
     def __init__(self, *args, **kwargs):
         AbstractNode.__init__(self, *args, **kwargs)
         self.links = {}
-    
+
     def render_links(self, check=False):
         """render the link attachments of the document"""
         assert not check, "Link checking not implemented"
@@ -132,7 +132,7 @@
                                'transition')
         for child in self.children:
             outcome.append(child.text())
-        
+
         # always a trailing newline
         text = self.sep.join([i for i in outcome if i]) + "\n"
         return text + self.render_links()
@@ -145,7 +145,7 @@
         self.char = char
         self.width = width
         super(Transition, self).__init__(*args, **kwargs)
-        
+
     def text(self):
         return (self.width - 1) * self.char
 
@@ -156,7 +156,7 @@
     sep = " "
     indent = ""
     width = 80
-    
+
     def __init__(self, *args, **kwargs):
         # make shortcut
         args = list(args)
@@ -164,19 +164,19 @@
             if isinstance(arg, str):
                 args[num] = Text(arg)
         super(Paragraph, self).__init__(*args, **kwargs)
-    
+
     def text(self):
         texts = []
         for child in self.children:
             texts += child.wordlist()
-        
+
         buf = []
         outcome = []
         lgt = len(self.indent)
-        
+
         def grab(buf):
             outcome.append(self.indent + self.sep.join(buf))
-        
+
         texts.reverse()
         while texts:
             next = texts[-1]
@@ -193,19 +193,19 @@
                 buf = []
         grab(buf)
         return "\n".join(outcome)
-    
+
 class SubParagraph(Paragraph):
     """ indented sub paragraph """
 
     indent = " "
-    
+
 class Title(Paragraph):
     """ title element """
 
     parentclass = Rest
     belowchar = "="
     abovechar = ""
-    
+
     def text(self):
         txt = self._get_text()
         lines = []
@@ -228,7 +228,7 @@
     end = ""
     def __init__(self, _text):
         self._text = _text
-    
+
     def text(self):
         text = self.escape(self._text)
         return self.start + text + self.end
@@ -241,7 +241,7 @@
         if self.end and self.end != self.start:
             text = text.replace(self.end, '\\%s' % (self.end,))
         return text
-    
+
 class Text(AbstractText):
     def wordlist(self):
         text = escape(self._text)
@@ -287,7 +287,7 @@
 class ListItem(Paragraph):
     allow_nesting = True
     item_chars = '*+-'
-    
+
     def text(self):
         idepth = self.get_indent_depth()
         indent = self.indent + (idepth + 1) * '  '
@@ -296,7 +296,7 @@
         item_char = self.item_chars[idepth]
         ret += [indent[len(item_char)+1:], item_char, ' ', txt[len(indent):]]
         return ''.join(ret)
-    
+
     def render_children(self, indent):
         txt = []
         buffer = []
@@ -352,7 +352,7 @@
         self._text = _text
         self.target = target
         self.rest = None
-    
+
     def text(self):
         if self.rest is None:
             self.rest = self.find_rest()
@@ -372,12 +372,12 @@
 class InternalLink(AbstractText):
     start = '`'
     end = '`_'
-    
+
 class LinkTarget(Paragraph):
     def __init__(self, name, target):
         self.name = name
         self.target = target
-    
+
     def text(self):
         return ".. _`%s`:%s\n" % (self.name, self.target)
 
@@ -392,7 +392,7 @@
         self.content = args
         super(Directive, self).__init__()
         self.options = options
-        
+
     def text(self):
         # XXX not very pretty...
         txt = '.. %s::' % (self.name,)
@@ -405,6 +405,6 @@
             txt += '\n'
             for item in self.content:
                 txt += '\n    ' + item
-        
+
         return txt
 
diff --git a/rpython/config/config.py b/rpython/config/config.py
--- a/rpython/config/config.py
+++ b/rpython/config/config.py
@@ -215,7 +215,7 @@
         self._name = name
         self.doc = doc
         self.cmdline = cmdline
-        
+
     def validate(self, value):
         raise NotImplementedError('abstract base class')
 
@@ -388,7 +388,7 @@
 
 class StrOption(Option):
     opt_type = 'string'
-    
+
     def __init__(self, name, doc, default=None, cmdline=DEFAULT_OPTION_NAME):
         super(StrOption, self).__init__(name, doc, cmdline)
         self.default = default
@@ -447,7 +447,7 @@
 
     def getpaths(self, include_groups=False, currpath=None):
         """returns a list of all paths in self, recursively
-        
+
             currpath should not be provided (helps with recursion)
         """
         if currpath is None:
@@ -492,15 +492,15 @@
                     defl = "default"
             else:
                 defl = "default: %s" % val
-                
+
         if option.type == 'choice':
             choices = option.choices
-            
+
         if choices is not None:
             choices = "%s=%s" % (option.metavar, '|'.join(choices))
         else:
             choices = ""
-        
+
         if '%default' in option.help:
             if choices and defl:
                 sep = ", "
@@ -511,7 +511,7 @@
                 defl = ""
             return option.help.replace("%default", defl)
         elif choices:
-            return option.help + ' [%s]' % choices 
+            return option.help + ' [%s]' % choices
 
         return option.help
 

From noreply at buildbot.pypy.org  Sun Oct 27 10:33:05 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 27 Oct 2013 10:33:05 +0100 (CET)
Subject: [pypy-commit] buildbot default: Never show entries more than two
 months old on the summary pages
Message-ID: <20131027093305.CA7181C00EC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r870:20116adf95a8
Date: 2013-10-27 10:32 +0100
http://bitbucket.org/pypy/buildbot/changeset/20116adf95a8/

Log:	Never show entries more than two months old on the summary pages

diff --git a/bot2/pypybuildbot/summary.py b/bot2/pypybuildbot/summary.py
--- a/bot2/pypybuildbot/summary.py
+++ b/bot2/pypybuildbot/summary.py
@@ -745,6 +745,8 @@
             for build in builditer:
                 if prune_old and self._age(build) > 7:
                     continue
+                if self._age(build) > 60:   # two months old: prune anyway
+                    continue
                 branch = self._get_branch(status, build)
                 if not test_branch(branch):
                     continue

From noreply at buildbot.pypy.org  Sun Oct 27 18:17:28 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 27 Oct 2013 18:17:28 +0100 (CET)
Subject: [pypy-commit] pypy default: Try to make the test pass on Win32
Message-ID: <20131027171728.C65CF1C02AE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67642:83885c43b10d
Date: 2013-10-27 18:16 +0100
http://bitbucket.org/pypy/pypy/changeset/83885c43b10d/

Log:	Try to make the test pass on Win32

diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -25,7 +25,8 @@
     def setup_method(self, meth):
         self.filepath = self.tmpdir.join(meth.im_func.func_name + '.py')
 
-    def run(self, func_or_src, args=[], import_site=False, **jitopts):
+    def run(self, func_or_src, args=[], import_site=False,
+            discard_stdout_before_last_line=False, **jitopts):
         jitopts.setdefault('threshold', 200)
         src = py.code.Source(func_or_src)
         if isinstance(func_or_src, types.FunctionType):
@@ -70,6 +71,9 @@
             stderr = ''
         assert not stderr
         #
+        if discard_stdout_before_last_line:
+            stdout = stdout.splitlines(True)[-1]
+        #
         # parse the JIT log
         rawlog = logparser.parse_log_file(str(logfile))
         rawtraces = logparser.extract_category(rawlog, 'jit-log-opt-')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -269,7 +269,8 @@
                 n += 1
             return n
 
-        log = self.run(main, [], import_site=True)
+        log = self.run(main, [], import_site=True,
+                       discard_stdout_before_last_line=True)  # <- for Win32
         assert log.result == 10000
         loop, = log.loops_by_id('cfficall')
         assert loop.match_by_id('cfficall', """

From noreply at buildbot.pypy.org  Mon Oct 28 17:27:03 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 28 Oct 2013 17:27:03 +0100 (CET)
Subject: [pypy-commit] pypy fix-trace-jit: Initial stab at fixing tracing
	with the JIT
Message-ID: <20131028162703.F07501C1066@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: fix-trace-jit
Changeset: r67643:ab50e9f78138
Date: 2013-10-28 09:26 -0700
http://bitbucket.org/pypy/pypy/changeset/ab50e9f78138/

Log:	Initial stab at fixing tracing with the JIT

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -10,7 +10,7 @@
 from rpython.rlib.rarithmetic import r_uint
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
-    UserDelAction, FrameTraceAction)
+    UserDelAction)
 from pypy.interpreter.error import (OperationError, operationerrfmt,
     new_exception_class)
 from pypy.interpreter.argument import Arguments
@@ -330,7 +330,6 @@
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
-        self.frame_trace_action = FrameTraceAction(self)
         self._code_of_sys_exc_info = None
 
         from pypy.interpreter.pycode import cpython_magic, default_magic
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -27,13 +27,10 @@
     def __init__(self, space):
         self.space = space
         self.topframeref = jit.vref_None
-        # tracing: space.frame_trace_action.fire() must be called to ensure
-        # that tracing occurs whenever self.w_tracefunc or self.is_tracing
-        # is modified.
-        self.w_tracefunc = None        # if not None, no JIT
+        self.w_tracefunc = None
         self.is_tracing = 0
         self.compiler = space.createcompiler()
-        self.profilefunc = None        # if not None, no JIT
+        self.profilefunc = None
         self.w_profilefuncarg = None
 
     def gettopframe(self):
@@ -76,9 +73,6 @@
                 frame_vref()
             jit.virtual_ref_finish(frame_vref, frame)
 
-        if self.gettrace() is not None and not frame.hide():
-            self.space.frame_trace_action.fire()
-
     # ________________________________________________________________
 
     def c_call_trace(self, frame, w_func, args=None):
@@ -123,25 +117,77 @@
     def return_trace(self, frame, w_retval):
         "Trace the return from a function"
         if self.gettrace() is not None:
-            return_from_hidden = self._trace(frame, 'return', w_retval)
-            # special case: if we are returning from a hidden function,
-            # then maybe we have to fire() the action again; otherwise
-            # it will not be called.  See test_trace_hidden_prints.
-            if return_from_hidden:
-                self.space.frame_trace_action.fire()
+            self._trace(frame, 'return', w_retval)
 
     def bytecode_trace(self, frame, decr_by=TICK_COUNTER_STEP):
         "Trace function called before each bytecode."
         # this is split into a fast path and a slower path that is
         # not invoked every time bytecode_trace() is.
+        self.bytecode_only_trace(frame)
         actionflag = self.space.actionflag
         if actionflag.decrement_ticker(decr_by) < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
     bytecode_trace._always_inline_ = True
 
+    def bytecode_only_trace(self, frame):
+        """
+        Like bytecode_trace() but doesn't invoke any other events besides the
+        trace function.
+        """
+        if (frame.w_f_trace is None or self.is_tracing or
+            self.gettrace() is None):
+            return
+        self.run_trace_func(frame)
+    bytecode_only_trace._always_inline_ = True
+
+    @jit.unroll_safe
+    def run_trace_func(self, frame):
+        code = frame.pycode
+        if frame.instr_lb <= frame.last_instr < frame.instr_ub:
+            if frame.last_instr < frame.instr_prev_plus_one:
+                # We jumped backwards in the same line.
+                self._trace(frame, 'line', self.space.w_None)
+        else:
+            size = len(code.co_lnotab) / 2
+            addr = 0
+            line = code.co_firstlineno
+            p = 0
+            lineno = code.co_lnotab
+            while size > 0:
+                c = ord(lineno[p])
+                if (addr + c) > frame.last_instr:
+                    break
+                addr += c
+                if c:
+                    frame.instr_lb = addr
+
+                line += ord(lineno[p + 1])
+                p += 2
+                size -= 1
+
+            if size > 0:
+                while True:
+                    size -= 1
+                    if size < 0:
+                        break
+                    addr += ord(lineno[p])
+                    if ord(lineno[p + 1]):
+                        break
+                    p += 2
+                frame.instr_ub = addr
+            else:
+                frame.instr_ub = sys.maxint
+
+            if frame.instr_lb == frame.last_instr: # At start of line!
+                frame.f_lineno = line
+                self._trace(frame, 'line', self.space.w_None)
+
+        frame.instr_prev_plus_one = frame.last_instr + 1
+
     def bytecode_trace_after_exception(self, frame):
         "Like bytecode_trace(), but without increasing the ticker."
         actionflag = self.space.actionflag
+        self.bytecode_only_trace(frame)
         if actionflag.get_ticker() < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
     bytecode_trace_after_exception._always_inline_ = 'try'
@@ -178,7 +224,6 @@
         else:
             self.force_all_frames()
             self.w_tracefunc = w_func
-            self.space.frame_trace_action.fire()
 
     def gettrace(self):
         return jit.promote(self.w_tracefunc)
@@ -221,7 +266,6 @@
         is_tracing = self.is_tracing
         self.is_tracing = 0
         try:
-            self.space.frame_trace_action.fire()
             return self.space.call(w_func, w_args)
         finally:
             self.is_tracing = is_tracing
@@ -260,7 +304,6 @@
             finally:
                 self.is_tracing -= 1
                 frame.locals2fast()
-                space.frame_trace_action.fire()
 
         # Profile cases
         if self.profilefunc is not None:
@@ -475,54 +518,3 @@
             except OperationError, e:
                 e.write_unraisable(space, descrname, w_obj)
                 e.clear(space)   # break up reference cycles
-
-class FrameTraceAction(AsyncAction):
-    """An action that calls the local trace functions (w_f_trace)."""
-
-    @jit.unroll_safe
-    def perform(self, executioncontext, frame):
-        if (frame.w_f_trace is None or executioncontext.is_tracing or
-            executioncontext.gettrace() is None):
-            return
-        code = frame.pycode
-        if frame.instr_lb <= frame.last_instr < frame.instr_ub:
-            if frame.last_instr < frame.instr_prev_plus_one:
-                # We jumped backwards in the same line.
-                executioncontext._trace(frame, 'line', self.space.w_None)
-        else:
-            size = len(code.co_lnotab) / 2
-            addr = 0
-            line = code.co_firstlineno
-            p = 0
-            lineno = code.co_lnotab
-            while size > 0:
-                c = ord(lineno[p])
-                if (addr + c) > frame.last_instr:
-                    break
-                addr += c
-                if c:
-                    frame.instr_lb = addr
-
-                line += ord(lineno[p + 1])
-                p += 2
-                size -= 1
-
-            if size > 0:
-                while True:
-                    size -= 1
-                    if size < 0:
-                        break
-                    addr += ord(lineno[p])
-                    if ord(lineno[p + 1]):
-                        break
-                    p += 2
-                frame.instr_ub = addr
-            else:
-                frame.instr_ub = sys.maxint
-
-            if frame.instr_lb == frame.last_instr: # At start of line!
-                frame.f_lineno = line
-                executioncontext._trace(frame, 'line', self.space.w_None)
-
-        frame.instr_prev_plus_one = frame.last_instr + 1
-        self.space.frame_trace_action.fire()     # continue tracing
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -464,8 +464,6 @@
         new_frame.instr_prev_plus_one = space.int_w(w_instr_prev_plus_one)
 
         self._setcellvars(cellvars)
-        # XXX what if the frame is in another thread??
-        space.frame_trace_action.fire()
 
     def hide(self):
         return self.pycode.hidden_applevel
@@ -759,7 +757,6 @@
         else:
             self.w_f_trace = w_trace
             self.f_lineno = self.get_last_lineno()
-            space.frame_trace_action.fire()
 
     def fdel_f_trace(self, space):
         self.w_f_trace = None
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -144,9 +144,11 @@
     def dispatch_bytecode(self, co_code, next_instr, ec):
         while True:
             self.last_instr = intmask(next_instr)
-            if not jit.we_are_jitted():
+            if jit.we_are_jitted():
+                ec.bytecode_only_trace(self)
+            else:
                 ec.bytecode_trace(self)
-                next_instr = r_uint(self.last_instr)
+            next_instr = r_uint(self.last_instr)
             opcode = ord(co_code[next_instr])
             next_instr += 1
 

From noreply at buildbot.pypy.org  Mon Oct 28 18:22:22 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 28 Oct 2013 18:22:22 +0100 (CET)
Subject: [pypy-commit] pypy default: Update the FAQ.
Message-ID: <20131028172222.C123C1C0163@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67644:bd931ba099f6
Date: 2013-10-28 18:21 +0100
http://bitbucket.org/pypy/pypy/changeset/bd931ba099f6/

Log:	Update the FAQ.

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -74,13 +74,19 @@
 On which platforms does PyPy run?
 ---------------------------------
 
-PyPy is regularly and extensively tested on Linux machines and on Mac
-OS X and mostly works under Windows too (but is tested there less
-extensively). PyPy needs a CPython running on the target platform to
-bootstrap, as cross compilation is not really meant to work yet.
-At the moment you need CPython 2.5 - 2.7
-for the translation process. PyPy's JIT requires an x86 or x86_64 CPU.
-(There has also been good progress on getting the JIT working for ARMv7.)
+PyPy is regularly and extensively tested on Linux machines. It mostly
+works on Mac and Windows: it is tested there, but most of us are running
+Linux so fixes may depend on 3rd-party contributions.  PyPy's JIT
+works on x86 (32-bit or 64-bit) and on ARM (ARMv6 or ARMv7).
+Support for POWER (64-bit) is stalled at the moment.
+
+To bootstrap from sources, PyPy can use either CPython (2.6 or 2.7) or
+another (e.g. older) PyPy.  Cross-translation is not really supported:
+e.g. to build a 32-bit PyPy, you need to have a 32-bit environment.
+Cross-translation is only explicitly supported between a 32-bit Intel
+Linux and ARM Linux (see here__).
+
+.. __: arm.html
 
 ------------------------------------------------
 Which Python version (2.x?) does PyPy implement?
@@ -125,7 +131,7 @@
 -----------------
 This really depends on your code.
 For pure Python algorithmic code, it is very fast.  For more typical
-Python programs we generally are 3 times the speed of Cpython 2.6 .
+Python programs we generally are 3 times the speed of CPython 2.7.
 You might be interested in our `benchmarking site`_ and our 
 `jit documentation`_.
 

From noreply at buildbot.pypy.org  Mon Oct 28 18:29:20 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 28 Oct 2013 18:29:20 +0100 (CET)
Subject: [pypy-commit] pypy fix-trace-jit: There's now another guard_isnull
	here
Message-ID: <20131028172920.1EE971C0163@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: fix-trace-jit
Changeset: r67645:b380ba48d968
Date: 2013-10-28 10:28 -0700
http://bitbucket.org/pypy/pypy/changeset/b380ba48d968/

Log:	There's now another guard_isnull here

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -425,6 +425,7 @@
         loop, = log.loops_by_id('call', is_entry_bridge=True)
         assert loop.match("""
             guard_value(i4, 1, descr=...)
+            guard_isnull(p5, descr=...)
             guard_nonnull_class(p12, ConstClass(W_IntObject), descr=...)
             guard_value(i8, 0, descr=...)
             guard_value(p2, ConstPtr(ptr21), descr=...)

From noreply at buildbot.pypy.org  Mon Oct 28 18:33:56 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 28 Oct 2013 18:33:56 +0100 (CET)
Subject: [pypy-commit] pypy default: Update this file
Message-ID: <20131028173356.2E2F31C0163@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67646:b6bf6ec537ac
Date: 2013-10-28 18:33 +0100
http://bitbucket.org/pypy/pypy/changeset/b6bf6ec537ac/

Log:	Update this file

diff --git a/pypy/doc/you-want-to-help.rst b/pypy/doc/you-want-to-help.rst
--- a/pypy/doc/you-want-to-help.rst
+++ b/pypy/doc/you-want-to-help.rst
@@ -44,17 +44,20 @@
   RPython to flow graphs and then to C. There is more in the `architecture`_
   document written about it.
 
-  It mostly lives in ``rpython``, ``annotator`` and ``objspace/flow``.
+  It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+  and ``rtyper``.
 
 .. _`architecture`: architecture.html 
 
-* Python Interpreter
+* Python Interpreter and modules
 
-  xxx
-
-* Python modules
-
-  xxx
+  This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
+  interpreter for Python written in RPython.  The fact that it is
+  RPython is not apparent at first.  Built-in modules are written in
+  ``pypy/module/*``.  Some modules that CPython implements in C are
+  simply written in pure Python; they are in the top-level ``lib_pypy``
+  directory.  The standard library of Python (with a few changes to
+  accomodate PyPy) is in ``lib-python``.
 
 * Just-in-Time Compiler (JIT): `we have a tracing JIT`_ that traces the
   interpreter written in RPython, rather than the user program that it
@@ -62,9 +65,9 @@
   language.  But getting it to work correctly is not trivial: it
   requires a small number of precise "hints" and possibly some small
   refactorings of the interpreter.  The JIT itself also has several
-  almost-independent parts: the tracer itself in ``jit/metainterp``, the
-  optimizer in ``jit/metainterp/optimizer`` that optimizes a list of
-  residual operations, and the backend in ``jit/backend/<machine-name>``
+  almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+  optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+  residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
   that turns it into machine code.  Writing a new backend is a
   traditional way to get into the project.
 
@@ -75,7 +78,7 @@
   `Garbage collection in PyPy`_ is inserted
   during translation.  Moreover, this is not reference counting; it is a real
   GC written as more RPython code.  The best one we have so far is in
-  ``rpython/memory/gc/minimark.py``.
+  ``rpython/memory/gc/incminimark.py``.
 
 .. _`Garbage collection in PyPy`: garbage_collection.html
 

From noreply at buildbot.pypy.org  Mon Oct 28 18:56:40 2013
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 28 Oct 2013 18:56:40 +0100 (CET)
Subject: [pypy-commit] pypy less-stringly-ops: unstringlify getitem_idx_key
	et al.
Message-ID: <20131028175640.3C4021C1054@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: less-stringly-ops
Changeset: r67648:21eaaa4e4ef3
Date: 2013-10-19 23:00 +0100
http://bitbucket.org/pypy/pypy/changeset/21eaaa4e4ef3/

Log:	unstringlify getitem_idx_key et al.

diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -266,6 +266,9 @@
 add_operator('setattr', 3, pyfunc=setattr)
 add_operator('delattr', 2, pyfunc=delattr)
 add_operator('getitem', 2, pure=True)
+add_operator('getitem_idx', 2, pure=True)
+add_operator('getitem_key', 2, pure=True)
+add_operator('getitem_idx_key', 2, pure=True)
 add_operator('setitem', 3)
 add_operator('delitem', 2)
 add_operator('getslice', 3, pyfunc=do_getslice, pure=True)
@@ -457,6 +460,9 @@
 # allows the annotator to be more precise, see test_reraiseAnything/KeyError in
 # the annotator tests
 op.getitem.canraise = [IndexError, KeyError, Exception]
+op.getitem_idx.canraise = [IndexError, KeyError, Exception]
+op.getitem_key.canraise = [IndexError, KeyError, Exception]
+op.getitem_idx_key.canraise = [IndexError, KeyError, Exception]
 op.setitem.canraise = [IndexError, KeyError, Exception]
 op.delitem.canraise = [IndexError, KeyError, Exception]
 op.contains.canraise = [Exception]    # from an r_dict
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -8,7 +8,7 @@
 
 from rpython.flowspace.model import (SpaceOperation, Variable, Constant,
                                      c_last_exception, checkgraph, mkentrymap)
-from rpython.flowspace.operation import OverflowingOperation
+from rpython.flowspace.operation import OverflowingOperation, op
 from rpython.rlib import rarithmetic
 from rpython.translator import unsimplify
 from rpython.translator.backendopt import ssa
@@ -204,7 +204,10 @@
                     elif exit.exitcase is KeyError:
                         postfx.append('key')
                 if postfx:
-                    last_op.opname = last_op.opname + '_' + '_'.join(postfx)
+                    Op = getattr(op, '_'.join(['getitem'] + postfx))
+                    newop = Op(*last_op.args)
+                    newop.result = last_op.result
+                    block.operations[-1] = newop
 
 
 def remove_dead_exceptions(graph):

From noreply at buildbot.pypy.org  Mon Oct 28 18:56:41 2013
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 28 Oct 2013 18:56:41 +0100 (CET)
Subject: [pypy-commit] pypy less-stringly-ops: Simplify
	ArgFT.unmatch_signature()
Message-ID: <20131028175641.55AB61C1066@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: less-stringly-ops
Changeset: r67649:2a3e02bd167a
Date: 2013-10-28 13:19 +0000
http://bitbucket.org/pypy/pypy/changeset/2a3e02bd167a/

Log:	Simplify ArgFT.unmatch_signature()

diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py
--- a/rpython/annotator/argument.py
+++ b/rpython/annotator/argument.py
@@ -189,37 +189,26 @@
 
     def unmatch_signature(self, signature, data_w):
         """kind of inverse of match_signature"""
-        need_cnt = len(self.positional_args)
-        need_kwds = self.keywords or []
-        space = self.space
         argnames, varargname, kwargname = signature
         assert kwargname is None
         cnt = len(argnames)
-        data_args_w = data_w[:cnt]
+        need_cnt = len(self.positional_args)
         if varargname:
-            data_w_stararg = data_w[cnt]
-            cnt += 1
-        else:
-            data_w_stararg = space.newtuple([])
+            assert len(data_w) == cnt + 1
+            stararg_w = self.space.unpackiterable(data_w[cnt])
+            if stararg_w:
+                args_w = data_w[:cnt] + stararg_w
+                assert len(args_w) == need_cnt
+                assert not self.keywords
+                return ArgumentsForTranslation(self.space, args_w, [], [])
+            else:
+                data_w = data_w[:-1]
         assert len(data_w) == cnt
-
-        unfiltered_kwds_w = {}
-        if len(data_args_w) >= need_cnt:
-            args_w = data_args_w[:need_cnt]
-            for argname, w_arg in zip(argnames[need_cnt:], data_args_w[need_cnt:]):
-                unfiltered_kwds_w[argname] = w_arg
-            assert not space.bool(data_w_stararg)
-        else:
-            stararg_w = space.unpackiterable(data_w_stararg)
-            args_w = data_args_w + stararg_w
-            assert len(args_w) == need_cnt
-
-        keywords = []
-        keywords_w = []
-        for key in need_kwds:
-            keywords.append(key)
-            keywords_w.append(unfiltered_kwds_w[key])
-
+        assert len(data_w) >= need_cnt
+        args_w = data_w[:need_cnt]
+        _kwds_w = dict(zip(argnames[need_cnt:], data_w[need_cnt:]))
+        keywords = self.keywords or []
+        keywords_w = [_kwds_w[key] for key in keywords]
         return ArgumentsForTranslation(self.space, args_w, keywords, keywords_w)
 
     @staticmethod

From noreply at buildbot.pypy.org  Mon Oct 28 18:56:39 2013
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 28 Oct 2013 18:56:39 +0100 (CET)
Subject: [pypy-commit] pypy less-stringly-ops: create OverflowingOp
Message-ID: <20131028175639.0B81E1C1050@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: less-stringly-ops
Changeset: r67647:e7fe7dd9a842
Date: 2013-10-19 20:20 +0100
http://bitbucket.org/pypy/pypy/changeset/e7fe7dd9a842/

Log:	create OverflowingOp

diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -53,6 +53,7 @@
 class HLOperation(SpaceOperation):
     __metaclass__ = HLOperationMeta
     pure = False
+    can_overflow = False
 
     def __init__(self, *args):
         self.args = list(args)
@@ -116,12 +117,25 @@
                         # store operation with variable result instead
                         pass
 
+class OverflowingOperation(PureOperation):
+    can_overflow = True
+    def ovfchecked(self):
+        ovf = self.ovf_variant(*self.args)
+        ovf.offset = self.offset
+        return ovf
+
 
 def add_operator(name, arity, pyfunc=None, pure=False, ovf=False):
     operator_func = getattr(operator, name, None)
-    base_cls = PureOperation if pure else HLOperation
+    if ovf:
+        assert pure
+        base_cls = OverflowingOperation
+    elif pure:
+        base_cls = PureOperation
+    else:
+        base_cls = HLOperation
     cls = HLOperationMeta(name, (base_cls,), {'opname': name, 'arity': arity,
-                                   'can_overflow': ovf, 'canraise': []})
+                                              'canraise': []})
     if pyfunc is not None:
         func2op[pyfunc] = cls
     if operator_func:
@@ -134,6 +148,7 @@
         from rpython.rlib.rarithmetic import ovfcheck
         ovf_func = lambda *args: ovfcheck(cls.pyfunc(*args))
         add_operator(name + '_ovf', arity, pyfunc=ovf_func)
+        cls.ovf_variant = getattr(op, name + '_ovf')
 
 # ____________________________________________________________
 
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -6,9 +6,9 @@
 """
 import py
 
-from rpython.flowspace import operation
 from rpython.flowspace.model import (SpaceOperation, Variable, Constant,
                                      c_last_exception, checkgraph, mkentrymap)
+from rpython.flowspace.operation import OverflowingOperation
 from rpython.rlib import rarithmetic
 from rpython.translator import unsimplify
 from rpython.translator.backendopt import ssa
@@ -92,13 +92,6 @@
     """
     covf = Constant(rarithmetic.ovfcheck)
 
-    def check_syntax(opname):
-        oper = getattr(operation.op, opname + "_ovf")
-        exlis = oper.canraise
-        if OverflowError not in exlis:
-            raise Exception("ovfcheck in %s: Operation %s has no"
-                            " overflow variant" % (graph.name, opname))
-
     for block in graph.iterblocks():
         for i in range(len(block.operations)-1, -1, -1):
             op = block.operations[i]
@@ -120,11 +113,14 @@
                     join_blocks(graph)         # merge the two blocks together
                     transform_ovfcheck(graph)  # ...and try again
                     return
-                op1 = block.operations[i-1]
-                check_syntax(op1.opname)
-                op1.opname += '_ovf'
+                op1 = block.operations[i - 1]
+                if not isinstance(op1, OverflowingOperation):
+                    raise Exception("ovfcheck in %s: Operation %s has no "
+                                    "overflow variant" % (graph.name, op1.opname))
+                op1_ovf = op1.ovfchecked()
+                block.operations[i - 1] = op1_ovf
                 del block.operations[i]
-                block.renamevariables({op.result: op1.result})
+                block.renamevariables({op.result: op1_ovf.result})
 
 def simplify_exceptions(graph):
     """The exception handling caused by non-implicit exceptions

From noreply at buildbot.pypy.org  Mon Oct 28 18:56:42 2013
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 28 Oct 2013 18:56:42 +0100 (CET)
Subject: [pypy-commit] pypy less-stringly-ops: remove unused class
	BasicAnnotatorPolicy
Message-ID: <20131028175642.8A93F1C1176@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: less-stringly-ops
Changeset: r67650:84054443ebdb
Date: 2013-10-28 14:37 +0000
http://bitbucket.org/pypy/pypy/changeset/84054443ebdb/

Log:	remove unused class BasicAnnotatorPolicy

diff --git a/rpython/annotator/policy.py b/rpython/annotator/policy.py
--- a/rpython/annotator/policy.py
+++ b/rpython/annotator/policy.py
@@ -1,31 +1,19 @@
 # base annotation policy for specialization
 from rpython.annotator.specialize import default_specialize as default
-from rpython.annotator.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype, specialize_arg_or_var
-from rpython.annotator.specialize import memo, specialize_call_location
+from rpython.annotator.specialize import (
+    specialize_argvalue, specialize_argtype, specialize_arglistitemtype,
+    specialize_arg_or_var, memo, specialize_call_location)
 
 
-class BasicAnnotatorPolicy(object):
+class AnnotatorPolicy(object):
+    """
+    Possibly subclass and pass an instance to the annotator to control
+    special-casing during annotation
+    """
 
     def event(pol, bookkeeper, what, *args):
         pass
 
-    def get_specializer(pol, tag):
-        return pol.no_specialization
-
-    def no_specialization(pol, funcdesc, args_s):
-        return funcdesc.cachedgraph(None)
-
-    def no_more_blocks_to_annotate(pol, annotator):
-        # hint to all pending specializers that we are done
-        for callback in annotator.bookkeeper.pending_specializations:
-            callback()
-        del annotator.bookkeeper.pending_specializations[:]
-
-class AnnotatorPolicy(BasicAnnotatorPolicy):
-    """
-    Possibly subclass and pass an instance to the annotator to control special casing during annotation
-    """
-
     def get_specializer(pol, directive):
         if directive is None:
             return pol.default_specialize
@@ -74,3 +62,9 @@
     def specialize__ll_and_arg(pol, *args):
         from rpython.rtyper.annlowlevel import LowLevelAnnotatorPolicy
         return LowLevelAnnotatorPolicy.specialize__ll_and_arg(*args)
+
+    def no_more_blocks_to_annotate(pol, annotator):
+        # hint to all pending specializers that we are done
+        for callback in annotator.bookkeeper.pending_specializations:
+            callback()
+        del annotator.bookkeeper.pending_specializations[:]

From noreply at buildbot.pypy.org  Mon Oct 28 18:56:43 2013
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 28 Oct 2013 18:56:43 +0100 (CET)
Subject: [pypy-commit] pypy less-stringly-ops: rm dead code in
	RpythonCallsSpace
Message-ID: <20131028175643.A8DFB1C1050@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: less-stringly-ops
Changeset: r67651:993ad988ef75
Date: 2013-10-20 01:37 +0100
http://bitbucket.org/pypy/pypy/changeset/993ad988ef75/

Log:	rm dead code in RpythonCallsSpace

diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py
--- a/rpython/annotator/argument.py
+++ b/rpython/annotator/argument.py
@@ -1,7 +1,7 @@
 """
 Arguments objects.
 """
-from rpython.annotator.model import SomeTuple, SomeObject
+from rpython.annotator.model import SomeTuple
 
 # for parsing call arguments
 class RPythonCallsSpace(object):
@@ -10,28 +10,11 @@
     that the call pattern is too complex for R-Python.
     """
     def newtuple(self, items_s):
-        if len(items_s) == 1 and items_s[0] is Ellipsis:
-            res = SomeObject()   # hack to get a SomeObject as the *arg
-            res.from_ellipsis = True
-            return res
-        else:
-            return SomeTuple(items_s)
+        return SomeTuple(items_s)
 
-    def unpackiterable(self, s_obj, expected_length=None):
-        if isinstance(s_obj, SomeTuple):
-            return list(s_obj.items)
-        if (s_obj.__class__ is SomeObject and
-            getattr(s_obj, 'from_ellipsis', False)):    # see newtuple()
-            return [Ellipsis]
-        raise CallPatternTooComplex("'*' argument must be SomeTuple")
-
-    def bool(self, s_tup):
-        assert isinstance(s_tup, SomeTuple)
-        return bool(s_tup.items)
-
-
-class CallPatternTooComplex(Exception):
-    pass
+    def unpackiterable(self, s_obj):
+        assert isinstance(s_obj, SomeTuple)
+        return list(s_obj.items)
 
 
 class ArgumentsForTranslation(object):
diff --git a/rpython/rtyper/callparse.py b/rpython/rtyper/callparse.py
--- a/rpython/rtyper/callparse.py
+++ b/rpython/rtyper/callparse.py
@@ -4,9 +4,6 @@
 from rpython.rtyper.error import TyperError
 from rpython.rtyper.lltypesystem import lltype
 
-class CallPatternTooComplex(TyperError):
-    pass
-
 
 def getrinputs(rtyper, graph):
     """Return the list of reprs of the input arguments to the 'graph'."""
@@ -163,27 +160,10 @@
     For the Arguments class: if it really needs other operations, it means
     that the call pattern is too complex for R-Python.
     """
-    w_tuple = NewTupleHolder
     def newtuple(self, items):
         return NewTupleHolder(items)
 
-    def newdict(self):
-        raise CallPatternTooComplex, "'**' argument"
-
-    def unpackiterable(self, it, expected_length=None):
-        if it.is_tuple():
-            items = it.items()
-            if (expected_length is not None and
-                expected_length != len(items)):
-                raise ValueError
-            return list(items)
-        raise CallPatternTooComplex, "'*' argument must be a tuple"
-    fixedview = unpackiterable
-    listview = unpackiterable
-
-    def is_w(self, one, other):
-        return one is other
-
-    def type(self, item):
-        return type(item)
-
+    def unpackiterable(self, it):
+        assert it.is_tuple()
+        items = it.items()
+        return list(items)

From noreply at buildbot.pypy.org  Mon Oct 28 18:56:44 2013
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon, 28 Oct 2013 18:56:44 +0100 (CET)
Subject: [pypy-commit] pypy less-stringly-ops: kill RPythonCallsSpace
Message-ID: <20131028175644.C9ED41C1050@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: less-stringly-ops
Changeset: r67652:40ebd6d2656d
Date: 2013-10-28 17:47 +0000
http://bitbucket.org/pypy/pypy/changeset/40ebd6d2656d/

Log:	kill RPythonCallsSpace

diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py
--- a/rpython/annotator/argument.py
+++ b/rpython/annotator/argument.py
@@ -3,27 +3,12 @@
 """
 from rpython.annotator.model import SomeTuple
 
-# for parsing call arguments
-class RPythonCallsSpace(object):
-    """Pseudo Object Space providing almost no real operation.
-    For the Arguments class: if it really needs other operations, it means
-    that the call pattern is too complex for R-Python.
-    """
-    def newtuple(self, items_s):
-        return SomeTuple(items_s)
-
-    def unpackiterable(self, s_obj):
-        assert isinstance(s_obj, SomeTuple)
-        return list(s_obj.items)
-
-
 class ArgumentsForTranslation(object):
     w_starstararg = None
-    def __init__(self, space, args_w, keywords=None, keywords_w=None,
+    def __init__(self, args_w, keywords=None, keywords_w=None,
                  w_stararg=None, w_starstararg=None):
         self.w_stararg = w_stararg
         assert w_starstararg is None
-        self.space = space
         assert isinstance(args_w, list)
         self.arguments_w = args_w
         self.keywords = keywords
@@ -42,11 +27,18 @@
     @property
     def positional_args(self):
         if self.w_stararg is not None:
-            args_w = self.space.unpackiterable(self.w_stararg)
+            args_w = self.unpackiterable(self.w_stararg)
             return self.arguments_w + args_w
         else:
             return self.arguments_w
 
+    def newtuple(self, items_s):
+        return SomeTuple(items_s)
+
+    def unpackiterable(self, s_obj):
+        assert isinstance(s_obj, SomeTuple)
+        return list(s_obj.items)
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -60,14 +52,13 @@
 
     def prepend(self, w_firstarg): # used often
         "Return a new Arguments with a new argument inserted first."
-        return ArgumentsForTranslation(self.space, [w_firstarg] + self.arguments_w,
+        return ArgumentsForTranslation([w_firstarg] + self.arguments_w,
                                        self.keywords, self.keywords_w, self.w_stararg,
                                        self.w_starstararg)
 
     def copy(self):
-        return ArgumentsForTranslation(self.space, self.arguments_w,
-                                       self.keywords, self.keywords_w, self.w_stararg,
-                                       self.w_starstararg)
+        return ArgumentsForTranslation(self.arguments_w, self.keywords,
+                self.keywords_w, self.w_stararg, self.w_starstararg)
 
     def _match_signature(self, scope_w, signature, defaults_w=None):
         """Parse args and kwargs according to the signature of a code object,
@@ -94,7 +85,7 @@
                 starargs_w = args_w[co_argcount:]
             else:
                 starargs_w = []
-            scope_w[co_argcount] = self.space.newtuple(starargs_w)
+            scope_w[co_argcount] = self.newtuple(starargs_w)
         elif num_args > co_argcount:
             raise ArgErrCount(num_args, num_kwds, signature, defaults_w, 0)
 
@@ -178,12 +169,12 @@
         need_cnt = len(self.positional_args)
         if varargname:
             assert len(data_w) == cnt + 1
-            stararg_w = self.space.unpackiterable(data_w[cnt])
+            stararg_w = self.unpackiterable(data_w[cnt])
             if stararg_w:
                 args_w = data_w[:cnt] + stararg_w
                 assert len(args_w) == need_cnt
                 assert not self.keywords
-                return ArgumentsForTranslation(self.space, args_w, [], [])
+                return ArgumentsForTranslation(args_w, [], [])
             else:
                 data_w = data_w[:-1]
         assert len(data_w) == cnt
@@ -192,10 +183,10 @@
         _kwds_w = dict(zip(argnames[need_cnt:], data_w[need_cnt:]))
         keywords = self.keywords or []
         keywords_w = [_kwds_w[key] for key in keywords]
-        return ArgumentsForTranslation(self.space, args_w, keywords, keywords_w)
+        return ArgumentsForTranslation(args_w, keywords, keywords_w)
 
-    @staticmethod
-    def fromshape(space, (shape_cnt, shape_keys, shape_star, shape_stst), data_w):
+    @classmethod
+    def fromshape(cls, (shape_cnt, shape_keys, shape_star, shape_stst), data_w):
         args_w = data_w[:shape_cnt]
         p = end_keys = shape_cnt + len(shape_keys)
         if shape_star:
@@ -208,9 +199,8 @@
             p += 1
         else:
             w_starstar = None
-        return ArgumentsForTranslation(space, args_w, list(shape_keys),
-                                       data_w[shape_cnt:end_keys], w_star,
-                                       w_starstar)
+        return cls(args_w, list(shape_keys), data_w[shape_cnt:end_keys],
+                w_star, w_starstar)
 
     def flatten(self):
         """ Argument <-> list of w_objects together with "shape" information """
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -18,7 +18,7 @@
 from rpython.annotator.dictdef import DictDef
 from rpython.annotator import description
 from rpython.annotator.signature import annotationoftype
-from rpython.annotator.argument import ArgumentsForTranslation, RPythonCallsSpace
+from rpython.annotator.argument import ArgumentsForTranslation
 from rpython.rlib.objectmodel import r_dict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
 from rpython.rtyper.lltypesystem import lltype, llmemory
@@ -692,12 +692,11 @@
         return op
 
     def build_args(self, op, args_s):
-        space = RPythonCallsSpace()
         if op == "simple_call":
-            return ArgumentsForTranslation(space, list(args_s))
+            return ArgumentsForTranslation(list(args_s))
         elif op == "call_args":
             return ArgumentsForTranslation.fromshape(
-                    space, args_s[0].const, # shape
+                    args_s[0].const, # shape
                     list(args_s[1:]))
 
     def ondegenerated(self, what, s_value, where=None, called_from_graph=None):
diff --git a/rpython/annotator/test/test_argument.py b/rpython/annotator/test/test_argument.py
--- a/rpython/annotator/test/test_argument.py
+++ b/rpython/annotator/test/test_argument.py
@@ -3,28 +3,23 @@
 from rpython.annotator.argument import ArgumentsForTranslation, rawshape
 from rpython.flowspace.argument import Signature
 
-class DummySpace(object):
+class MockArgs(ArgumentsForTranslation):
     def newtuple(self, items):
         return tuple(items)
 
-    def bool(self, obj):
-        return bool(obj)
-
     def unpackiterable(self, it):
         return list(it)
 
 
-def make_arguments_for_translation(space, args_w, keywords_w={},
-                                   w_stararg=None, w_starstararg=None):
-    return ArgumentsForTranslation(space, args_w, keywords_w.keys(),
-                                   keywords_w.values(), w_stararg,
-                                   w_starstararg)
+def make_arguments_for_translation(args_w, keywords_w={}, w_stararg=None,
+                                   w_starstararg=None):
+    return MockArgs(args_w, keywords_w.keys(), keywords_w.values(),
+                    w_stararg, w_starstararg)
 
 class TestArgumentsForTranslation(object):
 
     def test_prepend(self):
-        space = DummySpace()
-        args = ArgumentsForTranslation(space, ["0"])
+        args = MockArgs(["0"])
         args1 = args.prepend("thingy")
         assert args1 is not args
         assert args1.arguments_w == ["thingy", "0"]
@@ -32,12 +27,10 @@
         assert args1.keywords_w is args.keywords_w
 
     def test_fixedunpacked(self):
-        space = DummySpace()
-
-        args = ArgumentsForTranslation(space, [], ["k"], [1])
+        args = MockArgs([], ["k"], [1])
         py.test.raises(ValueError, args.fixedunpack, 1)
 
-        args = ArgumentsForTranslation(space, ["a", "b"])
+        args = MockArgs(["a", "b"])
         py.test.raises(ValueError, args.fixedunpack, 0)
         py.test.raises(ValueError, args.fixedunpack, 1)
         py.test.raises(ValueError, args.fixedunpack, 3)
@@ -46,122 +39,115 @@
         assert args.fixedunpack(2) == ['a', 'b']
 
     def test_unmatch_signature(self):
-        space = DummySpace()
-        args = make_arguments_for_translation(space, [1,2,3])
+        args = make_arguments_for_translation([1, 2, 3])
         sig = Signature(['a', 'b', 'c'], None, None)
         data = args.match_signature(sig, [])
         new_args = args.unmatch_signature(sig, data)
         assert args.unpack() == new_args.unpack()
 
-        args = make_arguments_for_translation(space, [1])
+        args = make_arguments_for_translation([1])
         sig = Signature(['a', 'b', 'c'], None, None)
         data = args.match_signature(sig, [2, 3])
         new_args = args.unmatch_signature(sig, data)
         assert args.unpack() == new_args.unpack()
 
-        args = make_arguments_for_translation(space, [1,2,3,4,5])
+        args = make_arguments_for_translation([1, 2, 3, 4, 5])
         sig = Signature(['a', 'b', 'c'], 'r', None)
         data = args.match_signature(sig, [])
         new_args = args.unmatch_signature(sig, data)
         assert args.unpack() == new_args.unpack()
 
-        args = make_arguments_for_translation(space, [1], {'c': 3, 'b': 2})
+        args = make_arguments_for_translation([1], {'c': 3, 'b': 2})
         sig = Signature(['a', 'b', 'c'], None, None)
         data = args.match_signature(sig, [])
         new_args = args.unmatch_signature(sig, data)
         assert args.unpack() == new_args.unpack()
 
-        args = make_arguments_for_translation(space, [1], {'c': 5})
+        args = make_arguments_for_translation([1], {'c': 5})
         sig = Signature(['a', 'b', 'c'], None, None)
         data = args.match_signature(sig, [2, 3])
         new_args = args.unmatch_signature(sig, data)
         assert args.unpack() == new_args.unpack()
 
     def test_rawshape(self):
-        space = DummySpace()
-        args = make_arguments_for_translation(space, [1,2,3])
+        args = make_arguments_for_translation([1, 2, 3])
         assert rawshape(args) == (3, (), False, False)
 
-        args = make_arguments_for_translation(space, [1])
+        args = make_arguments_for_translation([1])
         assert rawshape(args, 2) == (3, (), False, False)
 
-        args = make_arguments_for_translation(space, [1,2,3,4,5])
+        args = make_arguments_for_translation([1, 2, 3, 4, 5])
         assert rawshape(args) == (5, (), False, False)
 
-        args = make_arguments_for_translation(space, [1], {'c': 3, 'b': 2})
+        args = make_arguments_for_translation([1], {'c': 3, 'b': 2})
         assert rawshape(args) == (1, ('b', 'c'), False, False)
 
-        args = make_arguments_for_translation(space, [1], {'c': 5})
+        args = make_arguments_for_translation([1], {'c': 5})
         assert rawshape(args) == (1, ('c', ), False, False)
 
-        args = make_arguments_for_translation(space, [1], {'c': 5, 'd': 7})
+        args = make_arguments_for_translation([1], {'c': 5, 'd': 7})
         assert rawshape(args) == (1, ('c', 'd'), False, False)
 
-        args = make_arguments_for_translation(space, [1,2,3,4,5], {'e': 5, 'd': 7})
+        args = make_arguments_for_translation([1, 2, 3, 4, 5], {'e': 5, 'd': 7})
         assert rawshape(args) == (5, ('d', 'e'), False, False)
 
-
     def test_flatten(self):
-        space = DummySpace()
-        args = make_arguments_for_translation(space, [1,2,3])
+        args = make_arguments_for_translation([1, 2, 3])
         assert args.flatten() == ((3, (), False, False), [1, 2, 3])
 
-        args = make_arguments_for_translation(space, [1])
+        args = make_arguments_for_translation([1])
         assert args.flatten() == ((1, (), False, False), [1])
 
-        args = make_arguments_for_translation(space, [1,2,3,4,5])
-        assert args.flatten() == ((5, (), False, False), [1,2,3,4,5])
+        args = make_arguments_for_translation([1, 2, 3, 4, 5])
+        assert args.flatten() == ((5, (), False, False), [1, 2, 3, 4, 5])
 
-        args = make_arguments_for_translation(space, [1], {'c': 3, 'b': 2})
+        args = make_arguments_for_translation([1], {'c': 3, 'b': 2})
         assert args.flatten() == ((1, ('b', 'c'), False, False), [1, 2, 3])
 
-        args = make_arguments_for_translation(space, [1], {'c': 5})
+        args = make_arguments_for_translation([1], {'c': 5})
         assert args.flatten() == ((1, ('c', ), False, False), [1, 5])
 
-        args = make_arguments_for_translation(space, [1], {'c': 5, 'd': 7})
+        args = make_arguments_for_translation([1], {'c': 5, 'd': 7})
         assert args.flatten() == ((1, ('c', 'd'), False, False), [1, 5, 7])
 
-        args = make_arguments_for_translation(space, [1,2,3,4,5], {'e': 5, 'd': 7})
+        args = make_arguments_for_translation([1, 2, 3, 4, 5], {'e': 5, 'd': 7})
         assert args.flatten() == ((5, ('d', 'e'), False, False), [1, 2, 3, 4, 5, 7, 5])
 
     def test_stararg_flowspace_variable(self):
-        space = DummySpace()
         var = object()
         shape = ((2, ('g', ), True, False), [1, 2, 9, var])
-        args = make_arguments_for_translation(space, [1,2], {'g': 9},
+        args = make_arguments_for_translation([1, 2], {'g': 9},
                                        w_stararg=var)
         assert args.flatten() == shape
 
-        args = ArgumentsForTranslation.fromshape(space, *shape)
+        args = MockArgs.fromshape(*shape)
         assert args.flatten() == shape
 
-
     def test_fromshape(self):
-        space = DummySpace()
         shape = ((3, (), False, False), [1, 2, 3])
-        args = ArgumentsForTranslation.fromshape(space, *shape)
+        args = MockArgs.fromshape(*shape)
         assert args.flatten() == shape
 
         shape = ((1, (), False, False), [1])
-        args = ArgumentsForTranslation.fromshape(space, *shape)
+        args = MockArgs.fromshape(*shape)
         assert args.flatten() == shape
 
-        shape = ((5, (), False, False), [1,2,3,4,5])
-        args = ArgumentsForTranslation.fromshape(space, *shape)
+        shape = ((5, (), False, False), [1, 2, 3, 4, 5])
+        args = MockArgs.fromshape(*shape)
         assert args.flatten() == shape
 
         shape = ((1, ('b', 'c'), False, False), [1, 2, 3])
-        args = ArgumentsForTranslation.fromshape(space, *shape)
+        args = MockArgs.fromshape(*shape)
         assert args.flatten() == shape
 
         shape = ((1, ('c', ), False, False), [1, 5])
-        args = ArgumentsForTranslation.fromshape(space, *shape)
+        args = MockArgs.fromshape(*shape)
         assert args.flatten() == shape
 
         shape = ((1, ('c', 'd'), False, False), [1, 5, 7])
-        args = ArgumentsForTranslation.fromshape(space, *shape)
+        args = MockArgs.fromshape(*shape)
         assert args.flatten() == shape
 
         shape = ((5, ('d', 'e'), False, False), [1, 2, 3, 4, 5, 7, 5])
-        args = ArgumentsForTranslation.fromshape(space, *shape)
+        args = MockArgs.fromshape(*shape)
         assert args.flatten() == shape
diff --git a/rpython/rtyper/callparse.py b/rpython/rtyper/callparse.py
--- a/rpython/rtyper/callparse.py
+++ b/rpython/rtyper/callparse.py
@@ -4,6 +4,14 @@
 from rpython.rtyper.error import TyperError
 from rpython.rtyper.lltypesystem import lltype
 
+class ArgumentsForRtype(ArgumentsForTranslation):
+    def newtuple(self, items):
+        return NewTupleHolder(items)
+
+    def unpackiterable(self, it):
+        assert it.is_tuple()
+        items = it.items()
+        return list(items)
 
 def getrinputs(rtyper, graph):
     """Return the list of reprs of the input arguments to the 'graph'."""
@@ -27,7 +35,6 @@
     """Parse the arguments of 'hop' when calling the given 'graph'.
     """
     rinputs = getrinputs(rtyper, graph)
-    space = RPythonCallsSpace()
     def args_h(start):
         return [VarHolder(i, hop.args_s[i])
                         for i in range(start, hop.nb_args)]
@@ -37,9 +44,9 @@
         start = 0
         rinputs[0] = r_self
     if opname == "simple_call":
-        arguments =  ArgumentsForTranslation(space, args_h(start))
+        arguments =  ArgumentsForRtype(args_h(start))
     elif opname == "call_args":
-        arguments = ArgumentsForTranslation.fromshape(space,
+        arguments = ArgumentsForRtype.fromshape(
                 hop.args_s[start].const, # shape
                 args_h(start+1))
     # parse the arguments according to the function we are calling
@@ -153,17 +160,3 @@
         r_tup, v_tuple = self.holder.access(hop)
         v = r_tup.getitem_internal(hop, v_tuple, index)
         return hop.llops.convertvar(v, r_tup.items_r[index], repr)
-
-# for parsing call arguments
-class RPythonCallsSpace:
-    """Pseudo Object Space providing almost no real operation.
-    For the Arguments class: if it really needs other operations, it means
-    that the call pattern is too complex for R-Python.
-    """
-    def newtuple(self, items):
-        return NewTupleHolder(items)
-
-    def unpackiterable(self, it):
-        assert it.is_tuple()
-        items = it.items()
-        return list(items)
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -47,7 +47,7 @@
     hop = hop.copy()
     from rpython.annotator.argument import ArgumentsForTranslation
     arguments = ArgumentsForTranslation.fromshape(
-            None, hop.args_s[1].const, # shape
+            hop.args_s[1].const, # shape
             range(hop.nb_args-2))
     if arguments.w_starstararg is not None:
         raise TyperError("**kwds call not implemented")

From noreply at buildbot.pypy.org  Mon Oct 28 19:41:36 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 28 Oct 2013 19:41:36 +0100 (CET)
Subject: [pypy-commit] pypy fix-trace-jit: bump trace limit
Message-ID: <20131028184136.3F9741C00EC@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: fix-trace-jit
Changeset: r67653:59778ec85aa8
Date: 2013-10-28 11:40 -0700
http://bitbucket.org/pypy/pypy/changeset/59778ec85aa8/

Log:	bump trace limit

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -455,7 +455,7 @@
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
               'function_threshold': 1619, # slightly more than one above, also prime
               'trace_eagerness': 200,
-              'trace_limit': 6000,
+              'trace_limit': 8000,
               'inlining': 1,
               'loop_longevity': 1000,
               'retrace_limit': 5,

From noreply at buildbot.pypy.org  Mon Oct 28 22:07:51 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 28 Oct 2013 22:07:51 +0100 (CET)
Subject: [pypy-commit] pypy default: provide itemsize attribute for numpy
	scalars
Message-ID: <20131028210751.355EC1C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67654:e81d9905b8c7
Date: 2013-10-28 13:22 -0400
http://bitbucket.org/pypy/pypy/changeset/e81d9905b8c7/

Log:	provide itemsize attribute for numpy scalars

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -260,6 +260,9 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "view not implelemnted yet"))
 
+    def descr_get_itemsize(self, space):
+        return self.get_dtype(space).descr_get_itemsize(space)
+
 class W_BoolBox(W_GenericBox, PrimitiveBox):
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("bool")
 
@@ -508,6 +511,7 @@
     round = interp2app(W_GenericBox.descr_round),
     conjugate = interp2app(W_GenericBox.descr_conjugate),
     view = interp2app(W_GenericBox.descr_view),
+    itemsize = GetSetProperty(W_GenericBox.descr_get_itemsize),
 )
 
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
@@ -630,7 +634,7 @@
     __module__ = "numpypy",
     __new__ = interp2app(W_Complex64Box.descr__new__.im_func),
     __reduce__ = interp2app(W_Complex64Box.descr_reduce),
-    real = GetSetProperty(W_ComplexFloatingBox .descr_get_real),
+    real = GetSetProperty(W_ComplexFloatingBox.descr_get_real),
     imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
 )
 
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -37,3 +37,6 @@
         exc = raises(AttributeError, 'b.round()')
         assert exc.value[0] == "'bool' object has no attribute 'round'"
 
+    def test_itemsize(self):
+        import numpypy as np
+        assert np.int64(0).itemsize == 8

From noreply at buildbot.pypy.org  Mon Oct 28 22:07:52 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 28 Oct 2013 22:07:52 +0100 (CET)
Subject: [pypy-commit] pypy default: provide dtype attribute for numpy
	scalars
Message-ID: <20131028210752.935261C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67655:4e9e6df093b5
Date: 2013-10-28 13:38 -0400
http://bitbucket.org/pypy/pypy/changeset/4e9e6df093b5/

Log:	provide dtype attribute for numpy scalars

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -260,6 +260,9 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "view not implelemnted yet"))
 
+    def descr_get_dtype(self, space):
+        return self.get_dtype(space)
+
     def descr_get_itemsize(self, space):
         return self.get_dtype(space).descr_get_itemsize(space)
 
@@ -511,6 +514,8 @@
     round = interp2app(W_GenericBox.descr_round),
     conjugate = interp2app(W_GenericBox.descr_conjugate),
     view = interp2app(W_GenericBox.descr_view),
+
+    dtype = GetSetProperty(W_GenericBox.descr_get_dtype),
     itemsize = GetSetProperty(W_GenericBox.descr_get_itemsize),
 )
 
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -37,6 +37,7 @@
         exc = raises(AttributeError, 'b.round()')
         assert exc.value[0] == "'bool' object has no attribute 'round'"
 
-    def test_itemsize(self):
+    def test_attributes(self):
         import numpypy as np
+        assert np.int64(0).dtype == np.dtype('int64')
         assert np.int64(0).itemsize == 8

From noreply at buildbot.pypy.org  Mon Oct 28 22:07:53 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 28 Oct 2013 22:07:53 +0100 (CET)
Subject: [pypy-commit] pypy default: generalize numpy appbridge to pass
	through any arguments
Message-ID: <20131028210753.CBB701C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67656:21aee1bf4f2a
Date: 2013-10-28 13:25 -0400
http://bitbucket.org/pypy/pypy/changeset/21aee1bf4f2a/

Log:	generalize numpy appbridge to pass through any arguments

diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py
--- a/pypy/module/micronumpy/appbridge.py
+++ b/pypy/module/micronumpy/appbridge.py
@@ -1,4 +1,3 @@
-
 from rpython.rlib.objectmodel import specialize
 
 class AppBridgeCache(object):
@@ -16,16 +15,16 @@
             return sys.modules['numpypy.core._methods']
         return f
         """)
-    
+
     @specialize.arg(2)
-    def call_method(self, space, name, *args):
+    def call_method(self, space, name, w_obj, args):
         w_meth = getattr(self, 'w_' + name)
         if w_meth is None:
             if self.w_module is None:
                 self.w_module = space.call_function(self.w_import)
             w_meth = space.getattr(self.w_module, space.wrap(name))
             setattr(self, 'w_' + name, w_meth)
-        return space.call_function(w_meth, *args)
+        return space.call_args(w_meth, args.prepend(w_obj))
 
 def set_string_function(space, w_f, w_repr):
     cache = get_appbridge_cache(space)
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -827,15 +827,11 @@
         return loop.multidim_dot(space, self, other,  w_res, dtype,
                                  other_critical_dim)
 
-    @unwrap_spec(w_axis = WrappedDefault(None))
-    def descr_var(self, space, w_axis):
-        return get_appbridge_cache(space).call_method(space, '_var', self,
-                                                      w_axis)
+    def descr_var(self, space, __args__):
+        return get_appbridge_cache(space).call_method(space, '_var', self, __args__)
 
-    @unwrap_spec(w_axis = WrappedDefault(None))
-    def descr_std(self, space, w_axis):
-        return get_appbridge_cache(space).call_method(space, '_std', self,
-                                                      w_axis)
+    def descr_std(self, space, __args__):
+        return get_appbridge_cache(space).call_method(space, '_std', self, __args__)
 
     # ----------------------- reduce -------------------------------
 

From noreply at buildbot.pypy.org  Mon Oct 28 22:07:54 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 28 Oct 2013 22:07:54 +0100 (CET)
Subject: [pypy-commit] pypy default: delegate mean to numpy.core._methods
Message-ID: <20131028210754.E4CF51C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67657:5a3e53d3a04c
Date: 2013-10-28 13:29 -0400
http://bitbucket.org/pypy/pypy/changeset/5a3e53d3a04c/

Log:	delegate mean to numpy.core._methods

diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py
--- a/pypy/module/micronumpy/appbridge.py
+++ b/pypy/module/micronumpy/appbridge.py
@@ -1,6 +1,7 @@
 from rpython.rlib.objectmodel import specialize
 
 class AppBridgeCache(object):
+    w__mean = None
     w__var = None
     w__std = None
     w_module = None
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -827,6 +827,9 @@
         return loop.multidim_dot(space, self, other,  w_res, dtype,
                                  other_critical_dim)
 
+    def descr_mean(self, space, __args__):
+        return get_appbridge_cache(space).call_method(space, '_mean', self, __args__)
+
     def descr_var(self, space, __args__):
         return get_appbridge_cache(space).call_method(space, '_var', self, __args__)
 
@@ -862,14 +865,6 @@
     descr_cumsum = _reduce_ufunc_impl('add', cumultative=True)
     descr_cumprod = _reduce_ufunc_impl('multiply', cumultative=True)
 
-    def descr_mean(self, space, w_axis=None, w_out=None):
-        if space.is_none(w_axis):
-            w_denom = space.wrap(self.get_size())
-        else:
-            axis = unwrap_axis_arg(space, len(self.get_shape()), w_axis)
-            w_denom = space.wrap(self.get_shape()[axis])
-        return space.div(self.descr_sum_promote(space, w_axis, w_out), w_denom)
-
     def _reduce_argmax_argmin_impl(op_name):
         def impl(self, space):
             if self.get_size() == 0:

From noreply at buildbot.pypy.org  Mon Oct 28 22:07:56 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 28 Oct 2013 22:07:56 +0100 (CET)
Subject: [pypy-commit] pypy default: update numpy.core._methods from upstream
Message-ID: <20131028210756.353EB1C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67658:63bfeb2e635e
Date: 2013-10-28 13:43 -0400
http://bitbucket.org/pypy/pypy/changeset/63bfeb2e635e/

Log:	update numpy.core._methods from upstream

diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
--- a/lib_pypy/numpypy/core/_methods.py
+++ b/lib_pypy/numpypy/core/_methods.py
@@ -1,9 +1,16 @@
-# Array methods which are called by the both the C-code for the method
-# and the Python code for the NumPy-namespace function
+"""
+Array methods which are called by the both the C-code for the method
+and the Python code for the NumPy-namespace function
 
-import multiarray as mu
-import umath as um
-from numeric import asanyarray
+"""
+from __future__ import division, absolute_import, print_function
+
+import warnings
+
+from . import multiarray as mu
+from . import umath as um
+from .numeric import asanyarray
+from . import numerictypes as nt
 
 def _amax(a, axis=None, out=None, keepdims=False):
     return um.maximum.reduce(a, axis=axis,
@@ -31,7 +38,7 @@
 
 def _count_reduce_items(arr, axis):
     if axis is None:
-        axis = tuple(xrange(arr.ndim))
+        axis = tuple(range(arr.ndim))
     if not isinstance(axis, tuple):
         axis = (axis,)
     items = 1
@@ -42,58 +49,66 @@
 def _mean(a, axis=None, dtype=None, out=None, keepdims=False):
     arr = asanyarray(a)
 
-    # Upgrade bool, unsigned int, and int to float64
-    if dtype is None and arr.dtype.kind in ['b','u','i']:
-        ret = um.add.reduce(arr, axis=axis, dtype='f8',
-                            out=out, keepdims=keepdims)
+    rcount = _count_reduce_items(arr, axis)
+    # Make this warning show up first
+    if rcount == 0:
+        warnings.warn("Mean of empty slice.", RuntimeWarning)
+
+
+    # Cast bool, unsigned int, and int to float64 by default
+    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
+        dtype = mu.dtype('f8')
+
+    ret = um.add.reduce(arr, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+    if isinstance(ret, mu.ndarray):
+        ret = um.true_divide(
+                ret, rcount, out=ret, casting='unsafe', subok=False)
     else:
-        ret = um.add.reduce(arr, axis=axis, dtype=dtype,
-                            out=out, keepdims=keepdims)
-    rcount = _count_reduce_items(arr, axis)
-    if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(ret, rcount,
-                        out=ret, casting='unsafe', subok=False)
-    else:
-        ret = ret / float(rcount)
+        ret = ret.dtype.type(ret / rcount)
+
     return ret
 
-def _var(a, axis=None, dtype=None, out=None, ddof=0,
-                            keepdims=False):
+def _var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
     arr = asanyarray(a)
 
-    # First compute the mean, saving 'rcount' for reuse later
-    if dtype is None and arr.dtype.kind in ['b','u','i']:
-        arrmean = um.add.reduce(arr, axis=axis, dtype='f8', keepdims=True)
+    rcount = _count_reduce_items(arr, axis)
+    # Make this warning show up on top.
+    if ddof >= rcount:
+        warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning)
+
+    # Cast bool, unsigned int, and int to float64 by default
+    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
+        dtype = mu.dtype('f8')
+
+    # Compute the mean.
+    # Note that if dtype is not of inexact type then arraymean will
+    # not be either.
+    arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True)
+    if isinstance(arrmean, mu.ndarray):
+        arrmean = um.true_divide(
+                arrmean, rcount, out=arrmean, casting='unsafe', subok=False)
     else:
-        arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True)
-    rcount = _count_reduce_items(arr, axis)
-    if isinstance(arrmean, mu.ndarray):
-        arrmean = um.true_divide(arrmean, rcount,
-                            out=arrmean, casting='unsafe', subok=False)
-    else:
-        arrmean = arrmean / float(rcount)
+        arrmean = arrmean.dtype.type(arrmean / rcount)
 
-    # arr - arrmean
-    x = arr - arrmean
-
-    # (arr - arrmean) ** 2
-    if arr.dtype.kind == 'c':
+    # Compute sum of squared deviations from mean
+    # Note that x may not be inexact and that we need it to be an array,
+    # not a scalar.
+    x = asanyarray(arr - arrmean)
+    if issubclass(arr.dtype.type, nt.complexfloating):
         x = um.multiply(x, um.conjugate(x), out=x).real
     else:
         x = um.multiply(x, x, out=x)
-
-    # add.reduce((arr - arrmean) ** 2, axis)
     ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
 
-    # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof)
-    if not keepdims and isinstance(rcount, mu.ndarray):
-        rcount = rcount.squeeze(axis=axis)
-    rcount -= ddof
+    # Compute degrees of freedom and make sure it is not negative.
+    rcount = max([rcount - ddof, 0])
+
+    # divide by degrees of freedom
     if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(ret, rcount,
-                        out=ret, casting='unsafe', subok=False)
+        ret = um.true_divide(
+                ret, rcount, out=ret, casting='unsafe', subok=False)
     else:
-        ret = ret / float(rcount)
+        ret = ret.dtype.type(ret / rcount)
 
     return ret
 
@@ -104,6 +119,6 @@
     if isinstance(ret, mu.ndarray):
         ret = um.sqrt(ret, out=ret)
     else:
-        ret = um.sqrt(ret)
+        ret = ret.dtype.type(um.sqrt(ret))
 
     return ret
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1230,8 +1230,8 @@
         assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
         assert (arange(10).reshape(5, 2).mean(axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all()
         assert (a.mean(axis=-1) == a.mean(axis=2)).all()
-        raises(ValueError, a.mean, -4)
-        raises(ValueError, a.mean, 3)
+        raises(IndexError, a.mean, -4)
+        raises(IndexError, a.mean, 3)
         a = arange(10).reshape(5, 2)
         assert (a.mean(1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all()
 

From noreply at buildbot.pypy.org  Mon Oct 28 22:07:57 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 28 Oct 2013 22:07:57 +0100 (CET)
Subject: [pypy-commit] pypy default: remove unnecessary cast so this extra
 assert isn't needed
Message-ID: <20131028210757.54CDB1C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67659:b0865387a96d
Date: 2013-10-28 14:16 -0400
http://bitbucket.org/pypy/pypy/changeset/b0865387a96d/

Log:	remove unnecessary cast so this extra assert isn't needed

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -991,7 +991,7 @@
 
     def _read(self, storage, i, offset):
         res = raw_storage_getitem(self.T, storage, i + offset)
-        return rffi.cast(lltype.Float, byteswap(res))
+        return byteswap(res)
 
     def _write(self, storage, i, offset, value):
         swapped_value = byteswap(rffi.cast(self.T, value))
@@ -1052,14 +1052,6 @@
     BoxType = interp_boxes.W_Float32Box
     format_code = "f"
 
-    def read_bool(self, arr, i, offset):
-        # it's not clear to me why this is needed
-        # but a hint might be that calling for_computation(v)
-        # causes translation to fail, and the assert is necessary
-        v = self._read(arr.storage, i, offset)
-        assert isinstance(v, float)
-        return bool(v)
-
 class Float64(BaseType, Float):
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box

From noreply at buildbot.pypy.org  Mon Oct 28 22:07:58 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 28 Oct 2013 22:07:58 +0100 (CET)
Subject: [pypy-commit] pypy default: random cleanups for micronumpy module
Message-ID: <20131028210758.849291C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67660:06c60b3386a5
Date: 2013-10-28 14:29 -0400
http://bitbucket.org/pypy/pypy/changeset/06c60b3386a5/

Log:	random cleanups for micronumpy module

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -11,7 +11,6 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.tool.sourcetools import func_with_new_name
 from pypy.module.micronumpy.arrayimpl.voidbox import VoidBoxStorage
-from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rstring import StringBuilder
@@ -49,9 +48,6 @@
     _mixin_ = True
 
     def reduce(self, space):
-        from rpython.rlib.rstring import StringBuilder
-        from rpython.rtyper.lltypesystem import rffi, lltype
-
         numpypy = space.getbuiltinmodule("_numpypy")
         assert isinstance(numpypy, MixedModule)
         multiarray = numpypy.get("multiarray")
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,15 +1,14 @@
 from pypy.interpreter.error import operationerrfmt, OperationError
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr
-from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array,\
      ArrayArgumentException, issequence_w, wrap_impl
 from pypy.module.micronumpy import interp_dtype, interp_ufuncs, interp_boxes,\
-     interp_arrayops, iter
+     interp_arrayops
 from pypy.module.micronumpy.strides import find_shape_and_elems,\
      get_shape_from_iterable, to_coords, shape_agreement, \
      shape_agreement_multiple
 from pypy.module.micronumpy.interp_flatiter import W_FlatIterator
-from pypy.module.micronumpy.interp_support import unwrap_axis_arg
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy import loop
 from pypy.module.micronumpy.dot import match_dot_shapes
@@ -904,7 +903,6 @@
         raise OperationError(space.w_TypeError, space.wrap("only length-1 arrays can be converted to Python scalars"))
 
     def descr_reduce(self, space):
-        from rpython.rtyper.lltypesystem import rffi
         from rpython.rlib.rstring import StringBuilder
         from pypy.interpreter.mixedmodule import MixedModule
         from pypy.module.micronumpy.arrayimpl.concrete import SliceArray
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -173,7 +173,6 @@
         shapelen = len(obj_shape)
         axis = unwrap_axis_arg(space, shapelen, w_axis)
         assert axis >= 0
-        size = obj.get_size()
         dtype = interp_dtype.decode_w_dtype(space, dtype)
         if dtype is None:
             if self.comparison_func:
@@ -530,7 +529,6 @@
     int64_dtype = interp_dtype.get_dtype_cache(space).w_int64dtype
     complex_type = interp_dtype.get_dtype_cache(space).w_complex128dtype
     float_type = interp_dtype.get_dtype_cache(space).w_float64dtype
-    str_dtype = interp_dtype.get_dtype_cache(space).w_stringdtype
     if isinstance(w_obj, interp_boxes.W_GenericBox):
         dtype = w_obj.get_dtype(space)
         if current_guess is None:

From noreply at buildbot.pypy.org  Mon Oct 28 22:07:59 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 28 Oct 2013 22:07:59 +0100 (CET)
Subject: [pypy-commit] pypy default: update behavior for this numpy bool
 indexing special case
Message-ID: <20131028210759.AC4061C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67661:c07d42e46ab1
Date: 2013-10-28 14:49 -0400
http://bitbucket.org/pypy/pypy/changeset/c07d42e46ab1/

Log:	update behavior for this numpy bool indexing special case

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -94,12 +94,11 @@
             raise OperationError(space.w_ValueError,
                                  space.wrap("index out of range for array"))
         size = loop.count_all_true(idx)
-        if size > val.get_size() and val.get_size() > 1:
+        if size > val.get_size() and val.get_size() != 1:
             raise OperationError(space.w_ValueError, space.wrap("NumPy boolean array indexing assignment "
                                                                 "cannot assign %d input values to "
-                                                                "the %d output values where the mask is true" % (val.get_size(), size)))
-        if val.get_shape() == [0]:
-            val.implementation.dtype = self.implementation.dtype
+                                                                "the %d output values where the mask is true" %
+                                                                (val.get_size(), size)))
         loop.setitem_filter(self, idx, val, size)
 
     def _prepare_array_index(self, space, w_index):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2401,7 +2401,7 @@
     def test_array_indexing_bool_specialcases(self):
         from numpypy import arange, array
         a = arange(6)
-        exc = raises(ValueError,'a[a < 3] = [1, 2]')
+        exc = raises(ValueError, 'a[a < 3] = [1, 2]')
         assert exc.value[0].find('cannot assign') >= 0
         b = arange(4).reshape(2, 2) + 10
         a[a < 4] = b
@@ -2415,8 +2415,9 @@
         a[a > 3] = array([15])
         assert (a == [0, 1, 2, 3, 15, 15]).all()
         a = arange(6).reshape(3, 2)
-        a[a & 1 == 1] = []  # here, Numpy sticks garbage into the array
-        assert a.shape == (3, 2)
+        exc = raises(ValueError, 'a[a & 1 == 1] = []')
+        assert exc.value[0].find('cannot assign') >= 0
+        assert (a == [[0, 1], [2, 3], [4, 5]]).all()
 
     def test_copy_kwarg(self):
         from numpypy import array

From noreply at buildbot.pypy.org  Mon Oct 28 22:08:00 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 28 Oct 2013 22:08:00 +0100 (CET)
Subject: [pypy-commit] pypy default: be more strict about scalar any/all
	return types
Message-ID: <20131028210800.D6B921C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67662:840fdf1b5dac
Date: 2013-10-28 15:31 -0400
http://bitbucket.org/pypy/pypy/changeset/840fdf1b5dac/

Log:	be more strict about scalar any/all return types

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -229,12 +229,14 @@
         return space.newtuple([w_quotient, w_remainder])
 
     def descr_any(self, space):
+        from pypy.module.micronumpy.interp_dtype import get_dtype_cache
         value = space.is_true(self)
-        return self.get_dtype(space).box(value)
+        return get_dtype_cache(space).w_booldtype.box(value)
 
     def descr_all(self, space):
+        from pypy.module.micronumpy.interp_dtype import get_dtype_cache
         value = space.is_true(self)
-        return self.get_dtype(space).box(value)
+        return get_dtype_cache(space).w_booldtype.box(value)
 
     def descr_ravel(self, space):
         from pypy.module.micronumpy.base import convert_to_array
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -740,35 +740,25 @@
     def test_any_all_nonzero(self):
         import numpypy as numpy
         x = numpy.bool_(True)
-        assert x.any()
-        assert x.all()
-        assert x.__nonzero__()
-        assert isinstance(x.any(), numpy.bool_)
-        assert isinstance(x.__nonzero__(), bool)
+        assert x.any() is numpy.True_
+        assert x.all() is numpy.True_
+        assert x.__nonzero__() is True
         x = numpy.bool_(False)
-        assert not x.any()
-        assert not x.all()
-        assert not x.__nonzero__()
-        assert isinstance(x.any(), numpy.bool_)
-        assert isinstance(x.__nonzero__(), bool)
+        assert x.any() is numpy.False_
+        assert x.all() is numpy.False_
+        assert x.__nonzero__() is False
         x = numpy.float64(0)
-        assert not x.any()
-        assert not x.all()
-        assert not x.__nonzero__()
-        assert isinstance(x.any(), numpy.float64)
-        assert isinstance(x.__nonzero__(), bool)
+        assert x.any() is numpy.False_
+        assert x.all() is numpy.False_
+        assert x.__nonzero__() is False
         x = numpy.complex128(0)
-        assert not x.any()
-        assert not x.all()
-        assert not x.__nonzero__()
-        assert isinstance(x.any(), numpy.complex128)
-        assert isinstance(x.__nonzero__(), bool)
+        assert x.any() is numpy.False_
+        assert x.all() is numpy.False_
+        assert x.__nonzero__() is False
         x = numpy.complex128(0+1j)
-        assert x.any()
-        assert x.all()
-        assert x.__nonzero__()
-        assert isinstance(x.any(), numpy.complex128)
-        assert isinstance(x.__nonzero__(), bool)
+        assert x.any() is numpy.True_
+        assert x.all() is numpy.True_
+        assert x.__nonzero__() is True
 
     def test_ravel(self):
         from numpypy import float64, int8, array

From noreply at buildbot.pypy.org  Mon Oct 28 22:08:02 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 28 Oct 2013 22:08:02 +0100 (CET)
Subject: [pypy-commit] pypy default: cleanups for test_dtypes
Message-ID: <20131028210802.069CE1C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67663:9790fadf29d7
Date: 2013-10-28 15:45 -0400
http://bitbucket.org/pypy/pypy/changeset/9790fadf29d7/

Log:	cleanups for test_dtypes

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -17,7 +17,8 @@
 
 class AppTestDtypes(BaseAppTestDtypes):
     spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
-    def test_dtype(self):
+
+    def test_dtype_basic(self):
         from numpypy import dtype
 
         d = dtype('?')
@@ -190,36 +191,17 @@
             d3 = (array([1], d1) + array([1], d2)).dtype
             assert (d1, d2) == (d1, d2) and d3 is dtype(dout)
 
-    def test_add_int8(self):
-        from numpypy import array, dtype
-
-        a = array(range(5), dtype="int8")
-        b = a + a
-        assert b.dtype is dtype("int8")
-        for i in range(5):
-            assert b[i] == i * 2
-
-    def test_add_int16(self):
-        from numpypy import array, dtype
-
-        a = array(range(5), dtype="int16")
-        b = a + a
-        assert b.dtype is dtype("int16")
-        for i in range(5):
-            assert b[i] == i * 2
-
-    def test_add_uint32(self):
-        from numpypy import array, dtype
-
-        a = array(range(5), dtype="I")
-        b = a + a
-        assert b.dtype is dtype("I")
-        for i in range(5):
-            assert b[i] == i * 2
+    def test_add(self):
+        import numpypy as np
+        for dtype in ["int8", "int16", "I"]:
+            a = np.array(range(5), dtype=dtype)
+            b = a + a
+            assert b.dtype is np.dtype(dtype)
+            for i in range(5):
+                assert b[i] == i * 2
 
     def test_shape(self):
         from numpypy import dtype
-
         assert dtype(long).shape == ()
 
     def test_cant_subclass(self):
@@ -235,37 +217,15 @@
 
     def test_aliases(self):
         from numpypy import dtype
-
         assert dtype("float") is dtype(float)
 
-    def test_index_int8(self):
-        from numpypy import array, int8
-
-        a = array(range(10), dtype=int8)
-        b = array([0] * 10, dtype=int8)
-        for idx in b: a[idx] += 1
-
-    def test_index_int16(self):
-        from numpypy import array, int16
-
-        a = array(range(10), dtype=int16)
-        b = array([0] * 10, dtype=int16)
-        for idx in b: a[idx] += 1
-
-    def test_index_int32(self):
-        from numpypy import array, int32
-
-        a = array(range(10), dtype=int32)
-        b = array([0] * 10, dtype=int32)
-        for idx in b: a[idx] += 1
-
-    def test_index_int64(self):
-        from numpypy import array, int64
-
-        a = array(range(10), dtype=int64)
-        b = array([0] * 10, dtype=int64)
-        for idx in b:
-            a[idx] += 1
+    def test_index(self):
+        import numpypy as np
+        for dtype in [np.int8, np.int16, np.int32, np.int64]:
+            a = np.array(range(10), dtype=dtype)
+            b = np.array([0] * 10, dtype=dtype)
+            for idx in b:
+                a[idx] += 1
 
     def test_hash(self):
         import numpypy as numpy
@@ -770,7 +730,6 @@
         assert x.dtype == int8
         assert (x == array(42)).all()
 
-
 class AppTestStrUnicodeDtypes(BaseNumpyAppTest):
     def test_str_unicode(self):
         skip('numpypy differs from numpy')
@@ -971,5 +930,3 @@
         except NotImplementedError, e:
             if e.message.find('unable to create dtype from objects')>=0:
                 skip('creating ojbect dtype not supported yet')
-
-

From noreply at buildbot.pypy.org  Mon Oct 28 22:08:03 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 28 Oct 2013 22:08:03 +0100 (CET)
Subject: [pypy-commit] pypy default: clean up interp_dtype,
 store byteorder rather than native flag so we can differentiate
 between dtype(native) and dtype(natbyte)
Message-ID: <20131028210803.8DC111C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67664:0bb6ef2d4ba8
Date: 2013-10-28 16:17 -0400
http://bitbucket.org/pypy/pypy/changeset/0bb6ef2d4ba8/

Log:	clean up interp_dtype, store byteorder rather than native flag so we
	can differentiate between dtype(native) and dtype(natbyte)

diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -309,7 +309,7 @@
 def sort_array(arr, space, w_axis, w_order):
     cache = space.fromcache(SortCache) # that populates SortClasses
     itemtype = arr.dtype.itemtype
-    if not arr.dtype.native:
+    if not arr.dtype.is_native():
         raise OperationError(space.w_NotImplementedError,
            space.wrap("sorting of non-native btyeorder not supported yet"))
     for tp in all_types:
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -11,11 +11,11 @@
 from rpython.rlib import jit
 
 if sys.byteorder == 'little':
-    byteorder_prefix = '<'
-    nonnative_byteorder_prefix = '>'
+    NATBYTE = '<'
+    OPPBYTE = '>'
 else:
-    byteorder_prefix = '>'
-    nonnative_byteorder_prefix = '<'
+    NATBYTE = '>'
+    OPPBYTE = '<'
 
 UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
@@ -50,23 +50,23 @@
     return out
 
 class W_Dtype(W_Root):
-    _immutable_fields_ = ["itemtype", "num", "kind", "shape"]
+    _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder"]
 
-    def __init__(self, itemtype, num, kind, name, char, w_box_type,
+    def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder='=',
                  alternate_constructors=[], aliases=[], float_type=None,
-                 fields=None, fieldnames=None, native=True, shape=[], subdtype=None):
+                 fields=None, fieldnames=None, shape=[], subdtype=None):
         self.itemtype = itemtype
         self.num = num
         self.kind = kind
         self.name = name
         self.char = char
         self.w_box_type = w_box_type
+        self.byteorder = byteorder
         self.alternate_constructors = alternate_constructors
         self.aliases = aliases
         self.float_type = float_type
         self.fields = fields
         self.fieldnames = fieldnames
-        self.native = native
         self.shape = list(shape)
         self.subdtype = subdtype
         if not subdtype:
@@ -74,6 +74,11 @@
         else:
             self.base = subdtype.base
 
+    def __repr__(self):
+        if self.fields is not None:
+            return '<DType %r>' % self.fields
+        return '<DType %r>' % self.itemtype
+
     @specialize.argtype(1)
     def box(self, value):
         return self.itemtype.box(value)
@@ -101,6 +106,40 @@
     def fill(self, storage, box, start, stop):
         self.itemtype.fill(storage, self.get_size(), box, start, stop, 0)
 
+    def is_int_type(self):
+        return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or
+                self.kind == BOOLLTR)
+
+    def is_signed(self):
+        return self.kind == SIGNEDLTR
+
+    def is_complex_type(self):
+        return self.kind == COMPLEXLTR
+
+    def is_float_type(self):
+        return (self.kind == FLOATINGLTR or self.float_type is not None)
+
+    def is_bool_type(self):
+        return self.kind == BOOLLTR
+
+    def is_record_type(self):
+        return self.fields is not None
+
+    def is_str_type(self):
+        return self.num == 18
+
+    def is_str_or_unicode(self):
+        return (self.num == 18 or self.num == 19)
+
+    def is_flexible_type(self):
+        return (self.is_str_or_unicode() or self.is_record_type())
+
+    def is_native(self):
+        return self.byteorder in ('=', NATBYTE)
+
+    def get_size(self):
+        return self.itemtype.get_element_size()
+
     def get_name(self):
         if self.char == 'S':
             return '|S' + str(self.get_size())
@@ -115,37 +154,31 @@
     def descr_get_itemsize(self, space):
         return space.wrap(self.itemtype.get_element_size())
 
-    def descr_get_byteorder(self, space):
-        if self.native:
-            return space.wrap('=')
-        return space.wrap(nonnative_byteorder_prefix)
+    def descr_get_alignment(self, space):
+        return space.wrap(self.itemtype.alignment)
+
+    def descr_get_subdtype(self, space):
+        return space.newtuple([space.wrap(self.subdtype), self.descr_get_shape(space)])
 
     def descr_get_str(self, space):
         size = self.get_size()
         basic = self.kind
         if basic == UNICODELTR:
             size >>= 2
-            endian = byteorder_prefix
+            endian = NATBYTE
         elif size <= 1:
             endian = '|'  # ignore
-        elif self.native:
-            endian = byteorder_prefix
         else:
-            endian = nonnative_byteorder_prefix
-
+            endian = self.byteorder
+            if endian == '=':
+                endian = NATBYTE
         return space.wrap("%s%s%s" % (endian, basic, size))
 
-    def descr_get_alignment(self, space):
-        return space.wrap(self.itemtype.alignment)
-
-    def descr_get_isnative(self, space):
-        return space.wrap(self.native)
-
     def descr_get_base(self, space):
         return space.wrap(self.base)
 
-    def descr_get_subdtype(self, space):
-        return space.newtuple([space.wrap(self.subdtype), self.descr_get_shape(space)])
+    def descr_get_isnative(self, space):
+        return space.wrap(self.is_native())
 
     def descr_get_shape(self, space):
         w_shape = [space.wrap(dim) for dim in self.shape]
@@ -224,42 +257,6 @@
         except KeyError:
             raise OperationError(space.w_KeyError, space.wrap("Field named %s not found" % item))
 
-    def is_int_type(self):
-        return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or
-                self.kind == BOOLLTR)
-
-    def is_signed(self):
-        return self.kind == SIGNEDLTR
-
-    def is_complex_type(self):
-        return self.kind == COMPLEXLTR
-
-    def is_float_type(self):
-        return (self.kind == FLOATINGLTR or self.float_type is not None)
-
-    def is_bool_type(self):
-        return self.kind == BOOLLTR
-
-    def is_record_type(self):
-        return self.fields is not None
-
-    def is_str_type(self):
-        return self.num == 18
-
-    def is_str_or_unicode(self):
-        return (self.num == 18 or self.num == 19)
-
-    def is_flexible_type(self):
-        return (self.is_str_or_unicode() or self.is_record_type())
-
-    def __repr__(self):
-        if self.fields is not None:
-            return '<DType %r>' % self.fields
-        return '<DType %r>' % self.itemtype
-
-    def get_size(self):
-        return self.itemtype.get_element_size()
-
     def descr_reduce(self, space):
         w_class = space.type(self)
 
@@ -271,7 +268,7 @@
         names = self.descr_get_names(space)
         values = self.descr_get_fields(space)
         if self.fields:
-            order = space.wrap('|')
+            endian = '|'
             #TODO: Implement this when subarrays are implemented
             subdescr = space.w_None
             size = 0
@@ -283,21 +280,25 @@
             #TODO: Change this when alignment is implemented
             alignment = space.wrap(1)
         else:
-            order = space.wrap(byteorder_prefix if self.native else nonnative_byteorder_prefix)
+            endian = self.byteorder
+            if endian == '=':
+                endian = NATBYTE
             subdescr = space.w_None
             w_size = space.wrap(-1)
             alignment = space.wrap(-1)
         flags = space.wrap(0)
 
-        data = space.newtuple([version, order, subdescr, names, values, w_size, alignment, flags])
-
+        data = space.newtuple([version, space.wrap(endian), subdescr, names, values, w_size, alignment, flags])
         return space.newtuple([w_class, builder_args, data])
 
     def descr_setstate(self, space, w_data):
         if space.int_w(space.getitem(w_data, space.wrap(0))) != 3:
             raise OperationError(space.w_NotImplementedError, space.wrap("Pickling protocol version not supported"))
 
-        self.native = space.str_w(space.getitem(w_data, space.wrap(1))) == byteorder_prefix
+        endian = space.str_w(space.getitem(w_data, space.wrap(1)))
+        if endian == NATBYTE:
+            endian = '='
+        self.byteorder = endian
 
         fieldnames = space.getitem(w_data, space.wrap(3))
         self.set_names(space, fieldnames)
@@ -403,21 +404,22 @@
     __reduce__ = interp2app(W_Dtype.descr_reduce),
     __setstate__ = interp2app(W_Dtype.descr_setstate),
 
-    num = interp_attrproperty("num", cls=W_Dtype),
+    type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
     char = interp_attrproperty("char", cls=W_Dtype),
-    type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
-    byteorder = GetSetProperty(W_Dtype.descr_get_byteorder),
-    str = GetSetProperty(W_Dtype.descr_get_str),
+    num = interp_attrproperty("num", cls=W_Dtype),
+    byteorder = interp_attrproperty("byteorder", cls=W_Dtype),
     itemsize = GetSetProperty(W_Dtype.descr_get_itemsize),
     alignment = GetSetProperty(W_Dtype.descr_get_alignment),
+
+    subdtype = GetSetProperty(W_Dtype.descr_get_subdtype),
+    str = GetSetProperty(W_Dtype.descr_get_str),
+    name = interp_attrproperty("name", cls=W_Dtype),
+    base = GetSetProperty(W_Dtype.descr_get_base),
+    shape = GetSetProperty(W_Dtype.descr_get_shape),
     isnative = GetSetProperty(W_Dtype.descr_get_isnative),
-    shape = GetSetProperty(W_Dtype.descr_get_shape),
-    name = interp_attrproperty('name', cls=W_Dtype),
     fields = GetSetProperty(W_Dtype.descr_get_fields),
     names = GetSetProperty(W_Dtype.descr_get_names),
-    subdtype = GetSetProperty(W_Dtype.descr_get_subdtype),
-    base = GetSetProperty(W_Dtype.descr_get_base),
 )
 W_Dtype.typedef.acceptable_as_base_class = False
 
@@ -739,24 +741,24 @@
             self.dtypes_by_name[dtype.name] = dtype
             can_name = dtype.kind + str(dtype.itemtype.get_element_size())
             self.dtypes_by_name[can_name] = dtype
-            self.dtypes_by_name[byteorder_prefix + can_name] = dtype
+            self.dtypes_by_name[NATBYTE + can_name] = dtype
             self.dtypes_by_name['=' + can_name] = dtype
-            new_name = nonnative_byteorder_prefix + can_name
+            new_name = OPPBYTE + can_name
             itemtypename = dtype.itemtype.__class__.__name__
             itemtype = getattr(types, 'NonNative' + itemtypename)()
             self.dtypes_by_name[new_name] = W_Dtype(
                 itemtype,
                 dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                native=False)
+                byteorder=OPPBYTE)
             if dtype.kind != dtype.char:
                 can_name = dtype.char
-                self.dtypes_by_name[byteorder_prefix + can_name] = dtype
+                self.dtypes_by_name[NATBYTE + can_name] = dtype
                 self.dtypes_by_name['=' + can_name] = dtype
-                new_name = nonnative_byteorder_prefix + can_name
+                new_name = OPPBYTE + can_name
                 self.dtypes_by_name[new_name] = W_Dtype(
                     itemtype,
                     dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                    native=False)
+                    byteorder=OPPBYTE)
 
             for alias in dtype.aliases:
                 self.dtypes_by_name[alias] = dtype
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -472,7 +472,7 @@
         if self.is_scalar():
             return space.wrap(0)
         s = self.get_dtype().name
-        if not self.get_dtype().native:
+        if not self.get_dtype().is_native():
             s = s[1:]
         dtype = interp_dtype.get_dtype_cache(space).dtypes_by_name[s]
         contig = self.implementation.astype(space, dtype)
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,8 +1,7 @@
 from pypy.module.micronumpy.interp_dtype import get_dtype_cache
 from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
         find_unaryop_result_dtype)
-from pypy.module.micronumpy.interp_dtype import nonnative_byteorder_prefix,\
-     byteorder_prefix
+from pypy.module.micronumpy.interp_dtype import NATBYTE, OPPBYTE
 from pypy.conftest import option
 import sys
 
@@ -18,8 +17,8 @@
                 sys.modules['numpypy'] = numpy
                 isNumpy = True
         cls.w_isNumpy = cls.space.wrap(isNumpy)
-        cls.w_non_native_prefix = cls.space.wrap(nonnative_byteorder_prefix)
-        cls.w_native_prefix = cls.space.wrap(byteorder_prefix)
+        cls.w_non_native_prefix = cls.space.wrap(OPPBYTE)
+        cls.w_native_prefix = cls.space.wrap(NATBYTE)
 
 class TestUfuncCoerscion(object):
     def test_binops(self, space):

From noreply at buildbot.pypy.org  Mon Oct 28 22:14:47 2013
From: noreply at buildbot.pypy.org (necaris)
Date: Mon, 28 Oct 2013 22:14:47 +0100 (CET)
Subject: [pypy-commit] pypy improve-docs: Add notice re: building on OS X
Message-ID: <20131028211447.075E61C1050@cobra.cs.uni-duesseldorf.de>

Author: Rami Chowdhury <rami.chowdhury at gmail.com>
Branch: improve-docs
Changeset: r67665:b8141bb44cb1
Date: 2013-10-28 21:12 +0000
http://bitbucket.org/pypy/pypy/changeset/b8141bb44cb1/

Log:	Add notice re: building on OS X

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -86,6 +86,12 @@
     yum install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     lib-sqlite3-devel ncurses-devel expat-devel openssl-devel
 
+On Mac OS X, most of these build-time dependencies are installed alongside
+the Developer Tools. However, note that in order for the installation to
+find them you may need to run::
+
+    xcode-select --install
+
 
 Run the translation
 -------------------

From noreply at buildbot.pypy.org  Mon Oct 28 22:18:03 2013
From: noreply at buildbot.pypy.org (necaris)
Date: Mon, 28 Oct 2013 22:18:03 +0100 (CET)
Subject: [pypy-commit] pypy default: Add notice to docs re: building on OS X
Message-ID: <20131028211803.AD2A91C1050@cobra.cs.uni-duesseldorf.de>

Author: Rami Chowdhury <rami.chowdhury at gmail.com>
Branch: 
Changeset: r67666:df2902a1749e
Date: 2013-10-28 21:10 +0000
http://bitbucket.org/pypy/pypy/changeset/df2902a1749e/

Log:	Add notice to docs re: building on OS X

diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst
--- a/pypy/doc/getting-started-python.rst
+++ b/pypy/doc/getting-started-python.rst
@@ -63,6 +63,12 @@
      zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
      libexpat-devel libffi-devel python-curses
 
+   On Mac OS X, most of these build-time dependencies are installed alongside
+   the Developer Tools. However, note that in order for the installation to
+   find them you may need to run:
+
+     $ xcode-select --install
+
    The above command lines are split with continuation characters, giving the necessary dependencies first, then the optional ones.
 
    * ``pkg-config`` (to help us locate libffi files)

From noreply at buildbot.pypy.org  Mon Oct 28 22:18:05 2013
From: noreply at buildbot.pypy.org (necaris)
Date: Mon, 28 Oct 2013 22:18:05 +0100 (CET)
Subject: [pypy-commit] pypy default: Add notice re: building on OS X
Message-ID: <20131028211805.1017F1C1050@cobra.cs.uni-duesseldorf.de>

Author: Rami Chowdhury <rami.chowdhury at gmail.com>
Branch: 
Changeset: r67667:639304b40a0e
Date: 2013-10-28 21:17 +0000
http://bitbucket.org/pypy/pypy/changeset/639304b40a0e/

Log:	Add notice re: building on OS X

diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst
--- a/pypy/doc/getting-started-python.rst
+++ b/pypy/doc/getting-started-python.rst
@@ -63,6 +63,12 @@
      zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
      libexpat-devel libffi-devel python-curses
 
+   On Mac OS X, most of these build-time dependencies are installed alongside
+   the Developer Tools. However, note that in order for the installation to
+   find them you may need to run:
+
+     $ xcode-select --install
+
    The above command lines are split with continuation characters, giving the necessary dependencies first, then the optional ones.
 
    * ``pkg-config`` (to help us locate libffi files)

From noreply at buildbot.pypy.org  Mon Oct 28 23:05:53 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 28 Oct 2013 23:05:53 +0100 (CET)
Subject: [pypy-commit] pypy rordereddict: Import enough from
 rdict-experiments-3 in order to run direct tests
Message-ID: <20131028220553.6AE171C0163@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rordereddict
Changeset: r67668:9300a0671a5d
Date: 2013-10-28 20:19 +0200
http://bitbucket.org/pypy/pypy/changeset/9300a0671a5d/

Log:	Import enough from rdict-experiments-3 in order to run direct tests

diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -522,8 +522,10 @@
     A = lltype.typeOf(source)
     assert A == lltype.typeOf(dest)
     if isinstance(A.TO, lltype.GcArray):
-        assert isinstance(A.TO.OF, lltype.Ptr)
-        assert A.TO.OF.TO._gckind == 'gc'
+        if isinstance(A.TO.OF, lltype.Ptr):
+            assert A.TO.OF.TO._gckind == 'gc'
+        else:
+            assert isinstance(A.TO.OF, lltype.Struct)
     else:
         assert isinstance(A.TO, lltype.GcStruct)
         assert A.TO._arrayfld is not None
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -0,0 +1,1148 @@
+import sys
+from rpython.tool.pairtype import pairtype
+from rpython.flowspace.model import Constant
+from rpython.rtyper.rdict import AbstractDictRepr, AbstractDictIteratorRepr
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rlib import objectmodel, jit, rgc
+from rpython.rlib.debug import ll_assert
+from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rtyper import rmodel
+from rpython.rtyper.error import TyperError
+from rpython.rtyper.annlowlevel import llhelper
+
+
+# ____________________________________________________________
+#
+#  generic implementation of RPython dictionary, with parametric DICTKEY and
+#  DICTVALUE types. The basic implementation is a sparse array of indexes
+#  plus a dense array of structs that contain keys and values. struct looks
+#  like that:
+#
+#
+#    struct dictentry {
+#        DICTKEY key;
+#        DICTVALUE value;
+#        long f_hash;        # (optional) key hash, if hard to recompute
+#        bool f_valid;      # (optional) the entry is filled
+#    }
+#
+#    struct dicttable {
+#        int num_items;
+#        int num_used_items;
+#        int resize_counter;
+#        {byte, short, int, long} *indexes;
+#        dictentry *entries;
+#        lookup_function_no; # one of the four possible functions for different
+#                         # size dicts
+#        (Function DICTKEY, DICTKEY -> bool) *fnkeyeq;
+#        (Function DICTKEY -> int) *fnkeyhash;
+#    }
+#
+#
+
+def ll_call_lookup_function(d, key, hash, flag):
+    DICT = lltype.typeOf(d).TO
+    fun = d.lookup_function_no
+    if fun == FUNC_BYTE:
+        return DICT.lookup_family.byte_lookup_function(d, key, hash, flag)
+    elif fun == FUNC_SHORT:
+        return DICT.lookup_family.short_lookup_function(d, key, hash, flag)
+    elif IS_64BIT and fun == FUNC_INT:
+        return DICT.lookup_family.int_lookup_function(d, key, hash, flag)
+    elif fun == FUNC_LONG:
+        return DICT.lookup_family.long_lookup_function(d, key, hash, flag)
+    assert False
+
+def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
+                ll_fasthash_function=None, ll_hash_function=None,
+                ll_eq_function=None, method_cache={},
+                dummykeyobj=None, dummyvalueobj=None, rtyper=None,
+                setup_lookup_funcs=True):
+    # get the actual DICT type. if DICT is None, it's created, otherwise
+    # forward reference is becoming DICT
+    if DICT is None:
+        DICT = lltype.GcForwardReference()
+    # compute the shape of the DICTENTRY structure
+    entryfields = []
+    entrymeths = {
+        'allocate': lltype.typeMethod(_ll_malloc_entries),
+        'delete': _ll_free_entries,
+        'must_clear_key':   (isinstance(DICTKEY, lltype.Ptr)
+                             and DICTKEY._needsgc()),
+        'must_clear_value': (isinstance(DICTVALUE, lltype.Ptr)
+                             and DICTVALUE._needsgc()),
+        }
+
+    # * the key
+    entryfields.append(("key", DICTKEY))
+
+    # * the state of the entry - trying to encode it as dummy objects
+    if dummykeyobj:
+        # all the state can be encoded in the key
+        entrymeths['dummy_obj'] = dummykeyobj
+        entrymeths['valid'] = ll_valid_from_key
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_key
+        # the key is overwritten by 'dummy' when the entry is deleted
+        entrymeths['must_clear_key'] = False
+
+    elif dummyvalueobj:
+        # all the state can be encoded in the value
+        entrymeths['dummy_obj'] = dummyvalueobj
+        entrymeths['valid'] = ll_valid_from_value
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_value
+        # value is overwritten by 'dummy' when entry is deleted
+        entrymeths['must_clear_value'] = False
+
+    else:
+        # we need a flag to know if the entry was ever used
+        entryfields.append(("f_valid", lltype.Bool))
+        entrymeths['valid'] = ll_valid_from_flag
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_flag
+
+    # * the value
+    entryfields.append(("value", DICTVALUE))
+
+    if ll_fasthash_function is None:
+        entryfields.append(("f_hash", lltype.Signed))
+        entrymeths['hash'] = ll_hash_from_cache
+    else:
+        entrymeths['hash'] = ll_hash_recomputed
+        entrymeths['fasthashfn'] = ll_fasthash_function
+
+    # Build the lltype data structures
+    DICTENTRY = lltype.Struct("dictentry", *entryfields)
+    DICTENTRYARRAY = lltype.GcArray(DICTENTRY,
+                                    adtmeths=entrymeths)
+    fields =          [ ("num_items", lltype.Signed),
+                        ("num_used_items", lltype.Signed),
+                        ("resize_counter", lltype.Signed),
+                        ("indexes", llmemory.GCREF),
+                        ("lookup_function_no", lltype.Signed),
+                        ("entries", lltype.Ptr(DICTENTRYARRAY)) ]
+    if get_custom_eq_hash is not None:
+        r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash()
+        fields.extend([ ("fnkeyeq", r_rdict_eqfn.lowleveltype),
+                        ("fnkeyhash", r_rdict_hashfn.lowleveltype) ])
+        adtmeths = {
+            'keyhash':        ll_keyhash_custom,
+            'keyeq':          ll_keyeq_custom,
+            'r_rdict_eqfn':   r_rdict_eqfn,
+            'r_rdict_hashfn': r_rdict_hashfn,
+            'paranoia':       True,
+            }
+    else:
+        # figure out which functions must be used to hash and compare
+        ll_keyhash = ll_hash_function
+        ll_keyeq = ll_eq_function
+        ll_keyhash = lltype.staticAdtMethod(ll_keyhash)
+        if ll_keyeq is not None:
+            ll_keyeq = lltype.staticAdtMethod(ll_keyeq)
+        adtmeths = {
+            'keyhash':  ll_keyhash,
+            'keyeq':    ll_keyeq,
+            'paranoia': False,
+            }
+    adtmeths['KEY']   = DICTKEY
+    adtmeths['VALUE'] = DICTVALUE
+    adtmeths['lookup_function'] = lltype.staticAdtMethod(ll_call_lookup_function)
+    adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
+
+    family = LookupFamily()
+    adtmeths['lookup_family'] = family
+
+    DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
+                                *fields))
+
+    family.empty_array = DICTENTRYARRAY.allocate(0)
+    if setup_lookup_funcs:
+        _setup_lookup_funcs(DICT, rtyper, family)
+    return DICT
+
+def _setup_lookup_funcs(DICT, rtyper, family):
+    DICTKEY = DICT.entries.TO.OF.key
+    LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY,
+                                              lltype.Signed, lltype.Signed],
+                                              lltype.Signed))
+
+
+    STORECLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
+                                                  lltype.Signed,
+                                                  lltype.Signed],
+                                                 lltype.Void))
+
+    for name, T in [('byte', rffi.UCHAR),
+                    ('short', rffi.USHORT),
+                    ('int', rffi.UINT),
+                    ('long', lltype.Unsigned)]:
+        if name == 'int' and not IS_64BIT:
+            continue
+        lookupfn, storecleanfn = new_lookup_functions(LOOKUP_FUNC,
+                                                      STORECLEAN_FUNC, T=T,
+                                                      rtyper=rtyper)
+        setattr(family, '%s_lookup_function' % name, lookupfn)
+        setattr(family, '%s_insert_clean_function' % name, storecleanfn)
+
+def llhelper_or_compile(rtyper, FUNCPTR, ll_func):
+    # the check is for pseudo rtyper from tests
+    if rtyper is None or not hasattr(rtyper, 'annotate_helper_fn'):
+        return llhelper(FUNCPTR, ll_func)
+    else:
+        return rtyper.annotate_helper_fn(ll_func, FUNCPTR.TO.ARGS)
+
+class LookupFamily:
+    def _freeze_(self):
+        return True
+
+
+class DictRepr(AbstractDictRepr):
+
+    def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
+                 custom_eq_hash=None):
+        self.rtyper = rtyper
+        self.finalized = False
+        self.DICT = lltype.GcForwardReference()
+        self.lowleveltype = lltype.Ptr(self.DICT)
+        self.custom_eq_hash = custom_eq_hash is not None
+        if not isinstance(key_repr, rmodel.Repr):  # not computed yet, done by setup()
+            assert callable(key_repr)
+            self._key_repr_computer = key_repr
+        else:
+            self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr)
+        if not isinstance(value_repr, rmodel.Repr):  # not computed yet, done by setup()
+            assert callable(value_repr)
+            self._value_repr_computer = value_repr
+        else:
+            self.external_value_repr, self.value_repr = self.pickrepr(value_repr)
+        self.dictkey = dictkey
+        self.dictvalue = dictvalue
+        self.dict_cache = {}
+        self._custom_eq_hash_repr = custom_eq_hash
+        # setup() needs to be called to finish this initialization
+
+    def _externalvsinternal(self, rtyper, item_repr):
+        return rmodel.externalvsinternal(self.rtyper, item_repr)
+
+    def _setup_repr(self):
+        if 'key_repr' not in self.__dict__:
+            key_repr = self._key_repr_computer()
+            self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr)
+        if 'value_repr' not in self.__dict__:
+            self.external_value_repr, self.value_repr = self.pickrepr(self._value_repr_computer())
+        if isinstance(self.DICT, lltype.GcForwardReference):
+            DICTKEY = self.key_repr.lowleveltype
+            DICTVALUE = self.value_repr.lowleveltype
+            # * we need an explicit flag if the key and the value is not
+            #   able to store dummy values
+            s_key   = self.dictkey.s_value
+            s_value = self.dictvalue.s_value
+            kwd = {}
+            if self.custom_eq_hash:
+                self.r_rdict_eqfn, self.r_rdict_hashfn = (
+                    self._custom_eq_hash_repr())
+                kwd['get_custom_eq_hash'] = self._custom_eq_hash_repr
+            else:
+                kwd['ll_hash_function'] = self.key_repr.get_ll_hash_function()
+                kwd['ll_eq_function'] = self.key_repr.get_ll_eq_function()
+                kwd['ll_fasthash_function'] = self.key_repr.get_ll_fasthash_function()
+            kwd['dummykeyobj'] = self.key_repr.get_ll_dummyval_obj(self.rtyper,
+                                                                   s_key)
+            kwd['dummyvalueobj'] = self.value_repr.get_ll_dummyval_obj(
+                self.rtyper, s_value)
+
+            kwd['setup_lookup_funcs'] = False
+            get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT,
+                        rtyper=self.rtyper, **kwd)
+
+    def _setup_repr_final(self):
+        if not self.finalized:
+            family = self.lowleveltype.TO.lookup_family
+            _setup_lookup_funcs(self.lowleveltype.TO, self.rtyper, family)
+            self.finalized = True
+
+
+    def convert_const(self, dictobj):
+        from rpython.rtyper.lltypesystem import llmemory
+        # get object from bound dict methods
+        #dictobj = getattr(dictobj, '__self__', dictobj)
+        if dictobj is None:
+            return lltype.nullptr(self.DICT)
+        if not isinstance(dictobj, (dict, objectmodel.r_dict)):
+            raise TypeError("expected a dict: %r" % (dictobj,))
+        try:
+            key = Constant(dictobj)
+            return self.dict_cache[key]
+        except KeyError:
+            self.setup()
+            self.setup_final()
+            l_dict = ll_newdict_size(self.DICT, len(dictobj))
+            self.dict_cache[key] = l_dict
+            r_key = self.key_repr
+            if r_key.lowleveltype == llmemory.Address:
+                raise TypeError("No prebuilt dicts of address keys")
+            r_value = self.value_repr
+            if isinstance(dictobj, objectmodel.r_dict):
+                if self.r_rdict_eqfn.lowleveltype != lltype.Void:
+                    l_fn = self.r_rdict_eqfn.convert_const(dictobj.key_eq)
+                    l_dict.fnkeyeq = l_fn
+                if self.r_rdict_hashfn.lowleveltype != lltype.Void:
+                    l_fn = self.r_rdict_hashfn.convert_const(dictobj.key_hash)
+                    l_dict.fnkeyhash = l_fn
+
+                for dictkeycontainer, dictvalue in dictobj._dict.items():
+                    llkey = r_key.convert_const(dictkeycontainer.key)
+                    llvalue = r_value.convert_const(dictvalue)
+                    _ll_dict_insertclean(l_dict, llkey, llvalue,
+                                         dictkeycontainer.hash)
+                return l_dict
+
+            else:
+                for dictkey, dictvalue in dictobj.items():
+                    llkey = r_key.convert_const(dictkey)
+                    llvalue = r_value.convert_const(dictvalue)
+                    _ll_dict_insertclean(l_dict, llkey, llvalue,
+                                         l_dict.keyhash(llkey))
+                return l_dict
+
+    def rtype_len(self, hop):
+        v_dict, = hop.inputargs(self)
+        return hop.gendirectcall(ll_dict_len, v_dict)
+
+    def rtype_bool(self, hop):
+        v_dict, = hop.inputargs(self)
+        return hop.gendirectcall(ll_dict_bool, v_dict)
+
+    def make_iterator_repr(self, *variant):
+        return DictIteratorRepr(self, *variant)
+
+    def rtype_method_get(self, hop):
+        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                 self.value_repr)
+        hop.exception_cannot_occur()
+        v_res = hop.gendirectcall(ll_dict_get, v_dict, v_key, v_default)
+        return self.recast_value(hop.llops, v_res)
+
+    def rtype_method_setdefault(self, hop):
+        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                 self.value_repr)
+        hop.exception_cannot_occur()
+        v_res = hop.gendirectcall(ll_dict_setdefault, v_dict, v_key, v_default)
+        return self.recast_value(hop.llops, v_res)
+
+    def rtype_method_copy(self, hop):
+        v_dict, = hop.inputargs(self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_copy, v_dict)
+
+    def rtype_method_update(self, hop):
+        v_dic1, v_dic2 = hop.inputargs(self, self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_update, v_dic1, v_dic2)
+
+    def _rtype_method_kvi(self, hop, ll_func):
+        v_dic, = hop.inputargs(self)
+        r_list = hop.r_result
+        cLIST = hop.inputconst(lltype.Void, r_list.lowleveltype.TO)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_func, cLIST, v_dic)
+
+    def rtype_method_keys(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_keys)
+
+    def rtype_method_values(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_values)
+
+    def rtype_method_items(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_items)
+
+    def rtype_method_iterkeys(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "keys").newiter(hop)
+
+    def rtype_method_itervalues(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "values").newiter(hop)
+
+    def rtype_method_iteritems(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "items").newiter(hop)
+
+    def rtype_method_clear(self, hop):
+        v_dict, = hop.inputargs(self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_clear, v_dict)
+
+    def rtype_method_popitem(self, hop):
+        v_dict, = hop.inputargs(self)
+        r_tuple = hop.r_result
+        cTUPLE = hop.inputconst(lltype.Void, r_tuple.lowleveltype)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_popitem, cTUPLE, v_dict)
+
+    def rtype_method_pop(self, hop):
+        if hop.nb_args == 2:
+            v_args = hop.inputargs(self, self.key_repr)
+            target = ll_dict_pop
+        elif hop.nb_args == 3:
+            v_args = hop.inputargs(self, self.key_repr, self.value_repr)
+            target = ll_dict_pop_default
+        hop.exception_is_here()
+        v_res = hop.gendirectcall(target, *v_args)
+        return self.recast_value(hop.llops, v_res)
+
+class __extend__(pairtype(DictRepr, rmodel.Repr)):
+
+    def rtype_getitem((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        if not r_dict.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)   # record that we know about it
+        hop.exception_is_here()
+        v_res = hop.gendirectcall(ll_dict_getitem, v_dict, v_key)
+        return r_dict.recast_value(hop.llops, v_res)
+
+    def rtype_delitem((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        if not r_dict.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)   # record that we know about it
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_delitem, v_dict, v_key)
+
+    def rtype_setitem((r_dict, r_key), hop):
+        v_dict, v_key, v_value = hop.inputargs(r_dict, r_dict.key_repr, r_dict.value_repr)
+        if r_dict.custom_eq_hash:
+            hop.exception_is_here()
+        else:
+            hop.exception_cannot_occur()
+        hop.gendirectcall(ll_dict_setitem, v_dict, v_key, v_value)
+
+    def rtype_contains((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_contains, v_dict, v_key)
+
+class __extend__(pairtype(DictRepr, DictRepr)):
+    def convert_from_to((r_dict1, r_dict2), v, llops):
+        # check that we don't convert from Dicts with
+        # different key/value types
+        if r_dict1.dictkey is None or r_dict2.dictkey is None:
+            return NotImplemented
+        if r_dict1.dictkey is not r_dict2.dictkey:
+            return NotImplemented
+        if r_dict1.dictvalue is None or r_dict2.dictvalue is None:
+            return NotImplemented
+        if r_dict1.dictvalue is not r_dict2.dictvalue:
+            return NotImplemented
+        return v
+
+# ____________________________________________________________
+#
+#  Low-level methods.  These can be run for testing, but are meant to
+#  be direct_call'ed from rtyped flow graphs, which means that they will
+#  get flowed and annotated, mostly with SomePtr.
+
+DICTINDEX_LONG = lltype.Ptr(lltype.GcArray(lltype.Unsigned))
+DICTINDEX_INT = lltype.Ptr(lltype.GcArray(rffi.UINT))
+DICTINDEX_SHORT = lltype.Ptr(lltype.GcArray(rffi.USHORT))
+DICTINDEX_BYTE = lltype.Ptr(lltype.GcArray(rffi.UCHAR))
+
+IS_64BIT = sys.maxint != 2 ** 31 - 1
+
+if IS_64BIT:
+    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG = range(4)
+else:
+    FUNC_BYTE, FUNC_SHORT, FUNC_LONG = range(3)
+
+def ll_malloc_indexes_and_choose_lookup(d, n):
+    if n <= 256:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_BYTE.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_BYTE
+    elif n <= 65536:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_SHORT.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_SHORT
+    elif IS_64BIT and n <= 2 ** 32:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_INT.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_INT
+    else:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_LONG.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_LONG
+
+def ll_call_insert_clean_function(d, hash, i):
+    DICT = lltype.typeOf(d).TO
+    if d.lookup_function_no == FUNC_BYTE:
+        DICT.lookup_family.byte_insert_clean_function(d, hash, i)
+    elif d.lookup_function_no == FUNC_SHORT:
+        DICT.lookup_family.short_insert_clean_function(d, hash, i)
+    elif IS_64BIT and d.lookup_function_no == FUNC_INT:
+        DICT.lookup_family.int_insert_clean_function(d, hash, i)
+    elif d.lookup_function_no == FUNC_LONG:
+        DICT.lookup_family.long_insert_clean_function(d, hash, i)
+    else:
+        assert False
+
+def ll_valid_from_flag(entries, i):
+    return entries[i].f_valid
+
+def ll_valid_from_key(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    return entries[i].key != dummy
+
+def ll_valid_from_value(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    return entries[i].value != dummy
+
+def ll_mark_deleted_in_flag(entries, i):
+    entries[i].f_valid = False
+
+def ll_mark_deleted_in_key(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    entries[i].key = dummy
+
+def ll_mark_deleted_in_value(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    entries[i].value = dummy
+
+def ll_hash_from_cache(entries, i):
+    return entries[i].f_hash
+
+def ll_hash_recomputed(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    return ENTRIES.fasthashfn(entries[i].key)
+
+def ll_keyhash_custom(d, key):
+    DICT = lltype.typeOf(d).TO
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+
+def ll_keyeq_custom(d, key1, key2):
+    DICT = lltype.typeOf(d).TO
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+
+def ll_dict_len(d):
+    return d.num_items
+
+def ll_dict_bool(d):
+    # check if a dict is True, allowing for None
+    return bool(d) and d.num_items != 0
+
+def ll_dict_getitem(d, key):
+    index = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
+    if index != -1:
+        return d.entries[index].value
+    else:
+        raise KeyError
+
+def ll_dict_setitem(d, key, value):
+    hash = d.keyhash(key)
+    index = d.lookup_function(d, key, hash, FLAG_STORE)
+    return _ll_dict_setitem_lookup_done(d, key, value, hash, index)
+
+# It may be safe to look inside always, it has a few branches though, and their
+# frequencies needs to be investigated.
+ at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
+def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    if i >= 0:
+        entry = d.entries[i]
+        entry.value = value
+    else:
+        if len(d.entries) == d.num_used_items:
+            if ll_dict_grow(d):
+                ll_call_insert_clean_function(d, hash, d.num_used_items)
+        entry = d.entries[d.num_used_items]
+        entry.key = key
+        entry.value = value
+        if hasattr(ENTRY, 'f_hash'):
+            entry.f_hash = hash
+        if hasattr(ENTRY, 'f_valid'):
+            entry.f_valid = True
+        d.num_used_items += 1
+        d.num_items += 1
+        rc = d.resize_counter - 3
+        if rc <= 0:
+            ll_dict_resize(d)
+            rc = d.resize_counter - 3
+            ll_assert(rc > 0, "ll_dict_resize failed?")
+        d.resize_counter = rc
+
+def _ll_dict_insertclean(d, key, value, hash):
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    ll_call_insert_clean_function(d, hash, d.num_used_items)
+    entry = d.entries[d.num_used_items]
+    entry.key = key
+    entry.value = value
+    if hasattr(ENTRY, 'f_hash'):
+        entry.f_hash = hash
+    if hasattr(ENTRY, 'f_valid'):
+        entry.f_valid = True
+    d.num_used_items += 1
+    d.num_items += 1
+    rc = d.resize_counter - 3
+    d.resize_counter = rc
+
+def _ll_len_of_d_indexes(d):
+    # xxx Haaaack: returns len(d.indexes).  Works independently of
+    # the exact type pointed to by d, using a forced cast...
+    return len(rffi.cast(DICTINDEX_BYTE, d.indexes))
+
+def _overallocate_entries_len(baselen):
+    # This over-allocates proportional to the list size, making room
+    # for additional growth.  The over-allocation is mild, but is
+    # enough to give linear-time amortized behavior over a long
+    # sequence of appends() in the presence of a poorly-performing
+    # system malloc().
+    # The growth pattern is:  0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
+    newsize = baselen + 1
+    if newsize < 9:
+        some = 3
+    else:
+        some = 6
+    some += newsize >> 3
+    return newsize + some
+
+ at jit.dont_look_inside
+def ll_dict_grow(d):
+    if d.num_items < d.num_used_items // 4:
+        ll_dict_remove_deleted_items(d)
+        return True
+
+    new_allocated = _overallocate_entries_len(len(d.entries))
+
+    # Detect an obscure case where the indexes numeric type is too
+    # small to store all the entry indexes
+    if (max(128, _ll_len_of_d_indexes(d)) - new_allocated
+                   < MIN_INDEXES_MINUS_ENTRIES):
+        ll_dict_remove_deleted_items(d)
+        return True
+
+    newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+    rgc.ll_arraycopy(d.entries, newitems, 0, 0, len(d.entries))
+    d.entries = newitems
+    return False
+
+def ll_dict_remove_deleted_items(d):
+    new_allocated = _overallocate_entries_len(d.num_items)
+    if new_allocated < len(d.entries) // 2:
+        newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+    else:
+        newitems = d.entries
+    #
+    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
+    isrc = 0
+    idst = 0
+    while isrc < len(d.entries):
+        if d.entries.valid(isrc):
+            src = d.entries[isrc]
+            dst = newitems[idst]
+            dst.key = src.key
+            dst.value = src.value
+            if hasattr(ENTRY, 'f_hash'):
+                dst.f_hash = src.f_hash
+            if hasattr(ENTRY, 'f_valid'):
+                assert src.f_valid
+                dst.f_valid = True
+            idst += 1
+        isrc += 1
+    d.entries = newitems
+    assert d.num_items == idst
+    d.num_used_items = idst
+
+    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+
+
+def ll_dict_delitem(d, key):
+    index = d.lookup_function(d, key, d.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        raise KeyError
+    _ll_dict_del(d, index)
+
+ at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
+def _ll_dict_del(d, index):
+    d.entries.mark_deleted(index)
+    d.num_items -= 1
+    # clear the key and the value if they are GC pointers
+    ENTRIES = lltype.typeOf(d.entries).TO
+    ENTRY = ENTRIES.OF
+    entry = d.entries[index]
+    if ENTRIES.must_clear_key:
+        entry.key = lltype.nullptr(ENTRY.key.TO)
+    if ENTRIES.must_clear_value:
+        entry.value = lltype.nullptr(ENTRY.value.TO)
+    #
+    # The rest is commented out: like CPython we no longer shrink the
+    # dictionary here.  It may shrink later if we try to append a number
+    # of new items to it.  Unsure if this behavior was designed in
+    # CPython or is accidental.  A design reason would be that if you
+    # delete all items in a dictionary (e.g. with a series of
+    # popitem()), then CPython avoids shrinking the table several times.
+    #num_entries = len(d.entries)
+    #if num_entries > DICT_INITSIZE and d.num_items <= num_entries / 4:
+    #    ll_dict_resize(d)
+    # A previous xxx: move the size checking and resize into a single
+    # call which is opaque to the JIT when the dict isn't virtual, to
+    # avoid extra branches.
+
+def ll_dict_resize(d):
+    # make a 'new_size' estimate and shrink it if there are many
+    # deleted entry markers.  See CPython for why it is a good idea to
+    # quadruple the dictionary size as long as it's not too big.
+    num_items = d.num_items
+    if num_items > 50000:
+        new_estimate = num_items * 2
+    else:
+        new_estimate = num_items * 4
+    new_size = DICT_INITSIZE
+    while new_size <= new_estimate:
+        new_size *= 2
+
+    if new_size < _ll_len_of_d_indexes(d):
+        ll_dict_remove_deleted_items(d)
+    else:
+        ll_dict_reindex(d, new_size)
+ll_dict_resize.oopspec = 'dict.resize(d)'
+
+def ll_dict_reindex(d, new_size):
+    ll_malloc_indexes_and_choose_lookup(d, new_size)
+    d.resize_counter = new_size * 2 - d.num_items * 3
+    assert d.resize_counter > 0
+    #
+    entries = d.entries
+    i = 0
+    while i < d.num_used_items:
+        if entries.valid(i):
+            hash = entries.hash(i)
+            ll_call_insert_clean_function(d, hash, i)
+        i += 1
+    #old_entries.delete() XXXX!
+
+# ------- a port of CPython's dictobject.c's lookdict implementation -------
+PERTURB_SHIFT = 5
+
+FREE = 0
+DELETED = 1
+VALID_OFFSET = 2
+MIN_INDEXES_MINUS_ENTRIES = VALID_OFFSET + 1
+
+FLAG_LOOKUP = 0
+FLAG_STORE = 1
+FLAG_DELETE = 2
+FLAG_DELETE_TRY_HARD = 3
+
+def new_lookup_functions(LOOKUP_FUNC, STORECLEAN_FUNC, T, rtyper=None):
+    INDEXES = lltype.Ptr(lltype.GcArray(T))
+
+    def ll_kill_something(d):
+        i = 0
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        while True:
+            index = rffi.cast(lltype.Signed, indexes[i])
+            if index >= VALID_OFFSET:
+                indexes[i] = rffi.cast(T, DELETED)
+                return index
+            i += 1
+
+    @jit.look_inside_iff(lambda d, key, hash, store_flag:
+                         jit.isvirtual(d) and jit.isconstant(key))
+    def ll_dict_lookup(d, key, hash, store_flag):
+        entries = d.entries
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        mask = len(indexes) - 1
+        i = r_uint(hash & mask)
+        # do the first try before any looping
+        ENTRIES = lltype.typeOf(entries).TO
+        direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
+        index = rffi.cast(lltype.Signed, indexes[intmask(i)])
+        if index >= VALID_OFFSET:
+            checkingkey = entries[index - VALID_OFFSET].key
+            if direct_compare and checkingkey == key:
+                if store_flag == FLAG_DELETE:
+                    indexes[i] = rffi.cast(T, DELETED)
+                return index - VALID_OFFSET   # found the entry
+            if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
+                # correct hash, maybe the key is e.g. a different pointer to
+                # an equal object
+                found = d.keyeq(checkingkey, key)
+                #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found)
+                if d.paranoia:
+                    if (entries != d.entries or lltype.cast_opaque_ptr(llmemory.GCREF, indexes) != d.indexes or
+                        not entries.valid(index - VALID_OFFSET) or
+                        entries[index - VALID_OFFSET].key != checkingkey):
+                        # the compare did major nasty stuff to the dict: start over
+                        return ll_dict_lookup(d, key, hash, store_flag)
+                if found:
+                    if store_flag == FLAG_DELETE:
+                        indexes[i] = rffi.cast(T, DELETED)
+                    return index - VALID_OFFSET
+            deletedslot = -1
+        elif index == DELETED:
+            deletedslot = intmask(i)
+        else:
+            # pristine entry -- lookup failed
+            if store_flag == FLAG_STORE:
+                indexes[i] = rffi.cast(T, d.num_used_items + VALID_OFFSET)
+            elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
+                return ll_kill_something(d)
+            return -1
+
+        # In the loop, a deleted entry (everused and not valid) is by far
+        # (factor of 100s) the least likely outcome, so test for that last.
+        perturb = r_uint(hash)
+        while 1:
+            # compute the next index using unsigned arithmetic
+            i = (i << 2) + i + perturb + 1
+            i = i & mask
+            index = rffi.cast(lltype.Signed, indexes[intmask(i)])
+            if index == FREE:
+                if store_flag == FLAG_STORE:
+                    if deletedslot == -1:
+                        deletedslot = intmask(i)
+                    indexes[deletedslot] = rffi.cast(T, d.num_used_items +
+                                                     VALID_OFFSET)
+                elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
+                    return ll_kill_something(d)
+                return -1
+            elif index >= VALID_OFFSET:
+                checkingkey = entries[index - VALID_OFFSET].key
+                if direct_compare and checkingkey == key:
+                    if store_flag == FLAG_DELETE:
+                        indexes[i] = rffi.cast(T, DELETED)
+                    return index - VALID_OFFSET   # found the entry
+                if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
+                    # correct hash, maybe the key is e.g. a different pointer to
+                    # an equal object
+                    found = d.keyeq(checkingkey, key)
+                    if d.paranoia:
+                        if (entries != d.entries or lltype.cast_opaque_ptr(llmemory.GCREF, indexes) != d.indexes or
+                            not entries.valid(index - VALID_OFFSET) or
+                            entries[index - VALID_OFFSET].key != checkingkey):
+                            # the compare did major nasty stuff to the dict: start over
+                            return ll_dict_lookup(d, key, hash, store_flag)
+                    if found:
+                        if store_flag == FLAG_DELETE:
+                            indexes[i] = rffi.cast(T, DELETED)
+                        return index - VALID_OFFSET
+            elif deletedslot == -1:
+                deletedslot = intmask(i)
+            perturb >>= PERTURB_SHIFT
+
+    def ll_dict_store_clean(d, hash, index):
+        # a simplified version of ll_dict_lookup() which assumes that the
+        # key is new, and the dictionary doesn't contain deleted entries.
+        # It only finds the next free slot for the given hash.
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        mask = len(indexes) - 1
+        i = r_uint(hash & mask)
+        perturb = r_uint(hash)
+        while rffi.cast(lltype.Signed, indexes[i]) != 0:
+            i = (i << 2) + i + perturb + 1
+            i = i & mask
+            perturb >>= PERTURB_SHIFT
+        indexes[i] = rffi.cast(T, index + VALID_OFFSET)
+
+    return (llhelper_or_compile(rtyper, LOOKUP_FUNC, ll_dict_lookup),
+            llhelper_or_compile(rtyper, STORECLEAN_FUNC, ll_dict_store_clean))
+
+# ____________________________________________________________
+#
+#  Irregular operations.
+
+DICT_INITSIZE = 8
+
+def ll_newdict(DICT):
+    d = DICT.allocate()
+    d.entries = DICT.lookup_family.empty_array
+    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = DICT_INITSIZE * 2
+    return d
+
+def ll_newdict_size(DICT, orig_length_estimate):
+    length_estimate = (orig_length_estimate // 2) * 3
+    n = DICT_INITSIZE
+    while n < length_estimate:
+        n *= 2
+    d = DICT.allocate()
+    d.entries = DICT.entries.TO.allocate(orig_length_estimate)
+    ll_malloc_indexes_and_choose_lookup(d, n)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = n * 2
+    return d
+
+# rpython.memory.lldict uses a dict based on Struct and Array
+# instead of GcStruct and GcArray, which is done by using different
+# 'allocate' and 'delete' adtmethod implementations than the ones below
+def _ll_malloc_dict(DICT):
+    return lltype.malloc(DICT)
+def _ll_malloc_entries(ENTRIES, n):
+    return lltype.malloc(ENTRIES, n, zero=True)
+def _ll_free_entries(entries):
+    pass
+
+
+def rtype_r_dict(hop):
+    r_dict = hop.r_result
+    if not r_dict.custom_eq_hash:
+        raise TyperError("r_dict() call does not return an r_dict instance")
+    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
+    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
+    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
+    hop.exception_cannot_occur()
+    v_result = hop.gendirectcall(ll_newdict, cDICT)
+    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
+        hop.genop('setfield', [v_result, cname, v_eqfn])
+    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
+        hop.genop('setfield', [v_result, cname, v_hashfn])
+    return v_result
+
+# ____________________________________________________________
+#
+#  Iteration.
+
+def get_ll_dictiter(DICTPTR):
+    return lltype.Ptr(lltype.GcStruct('dictiter',
+                                      ('dict', DICTPTR),
+                                      ('index', lltype.Signed)))
+
+class DictIteratorRepr(AbstractDictIteratorRepr):
+
+    def __init__(self, r_dict, variant="keys"):
+        self.r_dict = r_dict
+        self.variant = variant
+        self.lowleveltype = get_ll_dictiter(r_dict.lowleveltype)
+        self.ll_dictiter = ll_dictiter
+        self.ll_dictnext = ll_dictnext_group[variant]
+
+
+def ll_dictiter(ITERPTR, d):
+    iter = lltype.malloc(ITERPTR.TO)
+    iter.dict = d
+    iter.index = 0
+    return iter
+
+def _make_ll_dictnext(kind):
+    # make three versions of the following function: keys, values, items
+    @jit.look_inside_iff(lambda RETURNTYPE, iter: jit.isvirtual(iter)
+                         and (iter.dict is None or
+                              jit.isvirtual(iter.dict)))
+    @jit.oopspec("dictiter.next%s(iter)" % kind)
+    def ll_dictnext(RETURNTYPE, iter):
+        # note that RETURNTYPE is None for keys and values
+        dict = iter.dict
+        if not dict:
+            raise StopIteration
+
+        entries = dict.entries
+        index = iter.index
+        assert index >= 0
+        entries_len = dict.num_used_items
+        while index < entries_len:
+            entry = entries[index]
+            is_valid = entries.valid(index)
+            index = index + 1
+            if is_valid:
+                iter.index = index
+                if RETURNTYPE is lltype.Void:
+                    return None
+                elif kind == 'items':
+                    r = lltype.malloc(RETURNTYPE.TO)
+                    r.item0 = recast(RETURNTYPE.TO.item0, entry.key)
+                    r.item1 = recast(RETURNTYPE.TO.item1, entry.value)
+                    return r
+                elif kind == 'keys':
+                    return entry.key
+                elif kind == 'values':
+                    return entry.value
+
+        # clear the reference to the dict and prevent restarts
+        iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
+        raise StopIteration
+
+    return ll_dictnext
+
+ll_dictnext_group = {'keys'  : _make_ll_dictnext('keys'),
+                     'values': _make_ll_dictnext('values'),
+                     'items' : _make_ll_dictnext('items')}
+
+# _____________________________________________________________
+# methods
+
+def ll_dict_get(dict, key, default):
+    index = dict.lookup_function(dict, key, dict.keyhash(key), FLAG_LOOKUP)
+    if index == -1:
+        return default
+    else:
+        return dict.entries[index].value
+
+def ll_dict_setdefault(dict, key, default):
+    hash = dict.keyhash(key)
+    index = dict.lookup_function(dict, key, hash, FLAG_STORE)
+    if index == -1:
+        _ll_dict_setitem_lookup_done(dict, key, default, hash, -1)
+        return default
+    else:
+        return dict.entries[index].value
+
+def ll_dict_copy(dict):
+    DICT = lltype.typeOf(dict).TO
+    newdict = DICT.allocate()
+    newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
+
+    newdict.num_items = dict.num_items
+    newdict.num_used_items = dict.num_used_items
+    if hasattr(DICT, 'fnkeyeq'):
+        newdict.fnkeyeq = dict.fnkeyeq
+    if hasattr(DICT, 'fnkeyhash'):
+        newdict.fnkeyhash = dict.fnkeyhash
+
+    i = 0
+    while i < newdict.num_used_items:
+        d_entry = newdict.entries[i]
+        entry = dict.entries[i]
+        ENTRY = lltype.typeOf(newdict.entries).TO.OF
+        d_entry.key = entry.key
+        if hasattr(ENTRY, 'f_valid'):
+            d_entry.f_valid = entry.f_valid
+        d_entry.value = entry.value
+        if hasattr(ENTRY, 'f_hash'):
+            d_entry.f_hash = entry.f_hash
+        i += 1
+
+    ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
+    return newdict
+ll_dict_copy.oopspec = 'dict.copy(dict)'
+
+def ll_dict_clear(d):
+    if d.num_used_items == 0:
+        return
+    DICT = lltype.typeOf(d).TO
+    old_entries = d.entries
+    d.entries = DICT.lookup_family.empty_array
+    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = DICT_INITSIZE * 2
+    # old_entries.delete() XXX
+ll_dict_clear.oopspec = 'dict.clear(d)'
+
+def ll_dict_update(dic1, dic2):
+    i = 0
+    while i < dic2.num_used_items:
+        entries = dic2.entries
+        if entries.valid(i):
+            entry = entries[i]
+            hash = entries.hash(i)
+            key = entry.key
+            value = entry.value
+            index = dic1.lookup_function(dic1, key, hash, FLAG_STORE)
+            _ll_dict_setitem_lookup_done(dic1, key, value, hash, index)
+        i += 1
+ll_dict_update.oopspec = 'dict.update(dic1, dic2)'
+
+# this is an implementation of keys(), values() and items()
+# in a single function.
+# note that by specialization on func, three different
+# and very efficient functions are created.
+
+def recast(P, v):
+    if isinstance(P, lltype.Ptr):
+        return lltype.cast_pointer(P, v)
+    else:
+        return v
+
+def _make_ll_keys_values_items(kind):
+    def ll_kvi(LIST, dic):
+        res = LIST.ll_newlist(dic.num_items)
+        entries = dic.entries
+        dlen = dic.num_used_items
+        items = res.ll_items()
+        i = 0
+        p = 0
+        while i < dlen:
+            if entries.valid(i):
+                ELEM = lltype.typeOf(items).TO.OF
+                if ELEM is not lltype.Void:
+                    entry = entries[i]
+                    if kind == 'items':
+                        r = lltype.malloc(ELEM.TO)
+                        r.item0 = recast(ELEM.TO.item0, entry.key)
+                        r.item1 = recast(ELEM.TO.item1, entry.value)
+                        items[p] = r
+                    elif kind == 'keys':
+                        items[p] = recast(ELEM, entry.key)
+                    elif kind == 'values':
+                        items[p] = recast(ELEM, entry.value)
+                p += 1
+            i += 1
+        assert p == res.ll_length()
+        return res
+    ll_kvi.oopspec = 'dict.%s(dic)' % kind
+    return ll_kvi
+
+ll_dict_keys   = _make_ll_keys_values_items('keys')
+ll_dict_values = _make_ll_keys_values_items('values')
+ll_dict_items  = _make_ll_keys_values_items('items')
+
+def ll_dict_contains(d, key):
+    i = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
+    return i != -1
+
+def _ll_getnextitem(dic):
+    if dic.num_items == 0:
+        raise KeyError
+
+    entries = dic.entries
+
+    while True:
+        i = dic.num_used_items - 1
+        if entries.valid(i):
+            break
+        dic.num_used_items -= 1
+
+    key = entries[i].key
+    index = dic.lookup_function(dic, key, entries.hash(i),
+                                FLAG_DELETE_TRY_HARD)
+    # if the lookup function returned me a random strange thing,
+    # don't care about deleting the item
+    if index == dic.num_used_items - 1:
+        dic.num_used_items -= 1
+    else:
+        assert index != -1
+    return index
+
+def ll_dict_popitem(ELEM, dic):
+    i = _ll_getnextitem(dic)
+    entry = dic.entries[i]
+    r = lltype.malloc(ELEM.TO)
+    r.item0 = recast(ELEM.TO.item0, entry.key)
+    r.item1 = recast(ELEM.TO.item1, entry.value)
+    _ll_dict_del(dic, i)
+    return r
+
+def ll_dict_pop(dic, key):
+    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        raise KeyError
+    value = dic.entries[index].value
+    _ll_dict_del(dic, index)
+    return value
+
+def ll_dict_pop_default(dic, key, dfl):
+    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        return dfl
+    value = dic.entries[index].value
+    _ll_dict_del(dic, index)
+    return value
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -0,0 +1,254 @@
+
+import py
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import rordereddict, rstr
+from rpython.rlib.rarithmetic import intmask
+from rpython.rtyper.annlowlevel import llstr, hlstr
+
+
+def get_indexes(ll_d):
+    return ll_d.indexes._obj.container._as_ptr()
+
+def foreach_index(ll_d):
+    indexes = get_indexes(ll_d)
+    for i in range(len(indexes)):
+        yield rffi.cast(lltype.Signed, indexes[i])
+
+def count_items(ll_d, ITEM):
+    c = 0
+    for item in foreach_index(ll_d):
+        if item == ITEM:
+            c += 1
+    return c
+
+    
+class TestRDictDirect(object):
+    dummykeyobj = None
+    dummyvalueobj = None
+
+    def _get_str_dict(self):
+        # STR -> lltype.Signed
+        DICT = rordereddict.get_ll_dict(lltype.Ptr(rstr.STR), lltype.Signed,
+                                 ll_fasthash_function=rstr.LLHelpers.ll_strhash,
+                                 ll_hash_function=rstr.LLHelpers.ll_strhash,
+                                 ll_eq_function=rstr.LLHelpers.ll_streq,
+                                 dummykeyobj=self.dummykeyobj,
+                                 dummyvalueobj=self.dummyvalueobj)
+        return DICT
+
+    def test_dict_creation(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        lls = llstr("abc")
+        rordereddict.ll_dict_setitem(ll_d, lls, 13)
+        assert count_items(ll_d, rordereddict.FREE) == rordereddict.DICT_INITSIZE - 1
+        assert rordereddict.ll_dict_getitem(ll_d, llstr("abc")) == 13
+        assert rordereddict.ll_dict_getitem(ll_d, lls) == 13
+        rordereddict.ll_dict_setitem(ll_d, lls, 42)
+        assert rordereddict.ll_dict_getitem(ll_d, lls) == 42
+        rordereddict.ll_dict_setitem(ll_d, llstr("abc"), 43)
+        assert rordereddict.ll_dict_getitem(ll_d, lls) == 43
+
+    def test_dict_creation_2(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        llab = llstr("ab")
+        llb = llstr("b")
+        rordereddict.ll_dict_setitem(ll_d, llab, 1)
+        rordereddict.ll_dict_setitem(ll_d, llb, 2)
+        assert rordereddict.ll_dict_getitem(ll_d, llb) == 2
+
+    def test_dict_store_get(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        for i in range(20):
+            for j in range(i):
+                assert rordereddict.ll_dict_getitem(ll_d, llstr(str(j))) == j
+            rordereddict.ll_dict_setitem(ll_d, llstr(str(i)), i)
+        assert ll_d.num_items == 20
+        for i in range(20):
+            assert rordereddict.ll_dict_getitem(ll_d, llstr(str(i))) == i
+
+    def test_dict_store_get_del(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        for i in range(20):
+            for j in range(0, i, 2):
+                assert rordereddict.ll_dict_getitem(ll_d, llstr(str(j))) == j
+            rordereddict.ll_dict_setitem(ll_d, llstr(str(i)), i)
+            if i % 2 != 0:
+                rordereddict.ll_dict_delitem(ll_d, llstr(str(i)))
+        assert ll_d.num_items == 10
+        for i in range(0, 20, 2):
+            assert rordereddict.ll_dict_getitem(ll_d, llstr(str(i))) == i
+
+    def test_dict_del_lastitem(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        py.test.raises(KeyError, rordereddict.ll_dict_delitem, ll_d, llstr("abc"))
+        rordereddict.ll_dict_setitem(ll_d, llstr("abc"), 13)
+        py.test.raises(KeyError, rordereddict.ll_dict_delitem, ll_d, llstr("def"))
+        rordereddict.ll_dict_delitem(ll_d, llstr("abc"))
+        assert count_items(ll_d, rordereddict.FREE) == rordereddict.DICT_INITSIZE - 1
+        assert count_items(ll_d, rordereddict.DELETED) == 1
+        py.test.raises(KeyError, rordereddict.ll_dict_getitem, ll_d, llstr("abc"))
+
+    def test_dict_del_not_lastitem(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, llstr("abc"), 13)
+        rordereddict.ll_dict_setitem(ll_d, llstr("def"), 15)
+        rordereddict.ll_dict_delitem(ll_d, llstr("abc"))
+        assert count_items(ll_d, rordereddict.FREE) == rordereddict.DICT_INITSIZE - 2
+        assert count_items(ll_d, rordereddict.DELETED) == 1
+
+    def test_dict_resize(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, llstr("a"), 1)
+        rordereddict.ll_dict_setitem(ll_d, llstr("b"), 2)
+        rordereddict.ll_dict_setitem(ll_d, llstr("c"), 3)
+        rordereddict.ll_dict_setitem(ll_d, llstr("d"), 4)
+        assert len(get_indexes(ll_d)) == 8
+        rordereddict.ll_dict_setitem(ll_d, llstr("e"), 5)
+        rordereddict.ll_dict_setitem(ll_d, llstr("f"), 6)
+        assert len(get_indexes(ll_d)) == 32
+        for item in ['a', 'b', 'c', 'd', 'e', 'f']:
+            assert rordereddict.ll_dict_getitem(ll_d, llstr(item)) == ord(item) - ord('a') + 1
+
+    def test_dict_grow_cleanup(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        lls = llstr("a")
+        for i in range(40):
+            rordereddict.ll_dict_setitem(ll_d, lls, i)
+            rordereddict.ll_dict_delitem(ll_d, lls)
+        assert ll_d.num_used_items <= 10
+
+    def test_dict_iteration(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        rordereddict.ll_dict_setitem(ll_d, llstr("j"), 2)
+        ITER = rordereddict.get_ll_dictiter(lltype.Ptr(DICT))
+        ll_iter = rordereddict.ll_dictiter(ITER, ll_d)
+        ll_iterkeys = rordereddict.ll_dictnext_group['keys']
+        next = ll_iterkeys(lltype.Signed, ll_iter)
+        assert hlstr(next) == "k"
+        next = ll_iterkeys(lltype.Signed, ll_iter)
+        assert hlstr(next) == "j"
+        py.test.raises(StopIteration, ll_iterkeys, lltype.Signed, ll_iter)
+
+    def test_popitem(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        rordereddict.ll_dict_setitem(ll_d, llstr("j"), 2)
+        TUP = lltype.Ptr(lltype.GcStruct('x', ('item0', lltype.Ptr(rstr.STR)),
+                                              ('item1', lltype.Signed)))
+        ll_elem = rordereddict.ll_dict_popitem(TUP, ll_d)
+        assert hlstr(ll_elem.item0) == "j"
+        assert ll_elem.item1 == 2
+        ll_elem = rordereddict.ll_dict_popitem(TUP, ll_d)
+        assert hlstr(ll_elem.item0) == "k"
+        assert ll_elem.item1 == 1
+        py.test.raises(KeyError, rordereddict.ll_dict_popitem, TUP, ll_d)
+
+    def test_direct_enter_and_del(self):
+        def eq(a, b):
+            return a == b
+
+        DICT = rordereddict.get_ll_dict(lltype.Signed, lltype.Signed,
+                                 ll_fasthash_function=intmask,
+                                 ll_hash_function=intmask,
+                                 ll_eq_function=eq)
+        ll_d = rordereddict.ll_newdict(DICT)
+        numbers = [i * rordereddict.DICT_INITSIZE + 1 for i in range(8)]
+        for num in numbers:
+            rordereddict.ll_dict_setitem(ll_d, num, 1)
+            rordereddict.ll_dict_delitem(ll_d, num)
+            for k in foreach_index(ll_d):
+                assert k < rordereddict.VALID_OFFSET
+
+    def test_contains(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        assert rordereddict.ll_dict_contains(ll_d, llstr("k"))
+        assert not rordereddict.ll_dict_contains(ll_d, llstr("j"))
+
+    def test_clear(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        rordereddict.ll_dict_setitem(ll_d, llstr("j"), 1)
+        rordereddict.ll_dict_setitem(ll_d, llstr("l"), 1)
+        rordereddict.ll_dict_clear(ll_d)
+        assert ll_d.num_items == 0
+
+    def test_get(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        assert rordereddict.ll_dict_get(ll_d, llstr("k"), 32) == 1
+        assert rordereddict.ll_dict_get(ll_d, llstr("j"), 32) == 32
+
+    def test_setdefault(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        assert rordereddict.ll_dict_setdefault(ll_d, llstr("j"), 42) == 42
+        assert rordereddict.ll_dict_getitem(ll_d, llstr("j")) == 42
+        assert rordereddict.ll_dict_setdefault(ll_d, llstr("k"), 42) == 1
+        assert rordereddict.ll_dict_getitem(ll_d, llstr("k")) == 1
+
+    def test_copy(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
+        rordereddict.ll_dict_setitem(ll_d, llstr("j"), 2)
+        ll_d2 = rordereddict.ll_dict_copy(ll_d)
+        for ll_d3 in [ll_d, ll_d2]:
+            assert rordereddict.ll_dict_getitem(ll_d3, llstr("k")) == 1
+            assert rordereddict.ll_dict_get(ll_d3, llstr("j"), 42) == 2
+            assert rordereddict.ll_dict_get(ll_d3, llstr("i"), 42) == 42
+
+    def test_update(self):
+        DICT = self._get_str_dict()
+        ll_d1 = rordereddict.ll_newdict(DICT)
+        ll_d2 = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d1, llstr("k"), 5)
+        rordereddict.ll_dict_setitem(ll_d1, llstr("j"), 6)
+        rordereddict.ll_dict_setitem(ll_d2, llstr("i"), 7)
+        rordereddict.ll_dict_setitem(ll_d2, llstr("k"), 8)
+        rordereddict.ll_dict_update(ll_d1, ll_d2)
+        for key, value in [("k", 8), ("i", 7), ("j", 6)]:
+            assert rordereddict.ll_dict_getitem(ll_d1, llstr(key)) == value
+
+    def test_pop(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, llstr("k"), 5)
+        rordereddict.ll_dict_setitem(ll_d, llstr("j"), 6)
+        assert rordereddict.ll_dict_pop(ll_d, llstr("k")) == 5
+        assert rordereddict.ll_dict_pop(ll_d, llstr("j")) == 6
+        py.test.raises(KeyError, rordereddict.ll_dict_pop, ll_d, llstr("k"))
+        py.test.raises(KeyError, rordereddict.ll_dict_pop, ll_d, llstr("j"))
+
+    def test_pop_default(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, llstr("k"), 5)
+        rordereddict.ll_dict_setitem(ll_d, llstr("j"), 6)
+        assert rordereddict.ll_dict_pop_default(ll_d, llstr("k"), 42) == 5
+        assert rordereddict.ll_dict_pop_default(ll_d, llstr("j"), 41) == 6
+        assert rordereddict.ll_dict_pop_default(ll_d, llstr("k"), 40) == 40
+        assert rordereddict.ll_dict_pop_default(ll_d, llstr("j"), 39) == 39
+
+class TestRDictDirectDummyKey(TestRDictDirect):
+    class dummykeyobj:
+        ll_dummy_value = llstr("dupa")
+
+class TestRDictDirectDummyValue(TestRDictDirect):
+    class dummyvalueobj:
+        ll_dummy_value = -42

From noreply at buildbot.pypy.org  Mon Oct 28 23:05:54 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 28 Oct 2013 23:05:54 +0100 (CET)
Subject: [pypy-commit] pypy rordereddict: Provide OrderedDict implemention
 for RPython, stolen from rdict-experiments-3
Message-ID: <20131028220554.E2DC31C0163@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rordereddict
Changeset: r67669:9f44c1e921fa
Date: 2013-10-29 00:05 +0200
http://bitbucket.org/pypy/pypy/changeset/9f44c1e921fa/

Log:	Provide OrderedDict implemention for RPython, stolen from rdict-
	experiments-3 branch. Note that two new tests are skipped, because
	rdict-experiments-3 branch has the official version of this file

diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -6,7 +6,8 @@
 import operator
 from rpython.tool.pairtype import pair, pairtype
 from rpython.annotator.model import SomeObject, SomeInteger, SomeBool, s_Bool
-from rpython.annotator.model import SomeString, SomeChar, SomeList, SomeDict
+from rpython.annotator.model import SomeString, SomeChar, SomeList, SomeDict,\
+     SomeOrderedDict
 from rpython.annotator.model import SomeUnicodeCodePoint, SomeUnicodeString
 from rpython.annotator.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue
 from rpython.annotator.model import SomeInstance, SomeBuiltin, SomeIterator
@@ -581,7 +582,8 @@
 class __extend__(pairtype(SomeDict, SomeDict)):
 
     def union((dic1, dic2)):
-        return SomeDict(dic1.dictdef.union(dic2.dictdef))
+        assert dic1.__class__ == dic2.__class__
+        return dic1.__class__(dic1.dictdef.union(dic2.dictdef))
 
 
 class __extend__(pairtype(SomeDict, SomeObject)):
@@ -840,6 +842,7 @@
 _make_none_union('SomeString',      'no_nul=obj.no_nul, can_be_None=True')
 _make_none_union('SomeUnicodeString', 'can_be_None=True')
 _make_none_union('SomeList',         'obj.listdef')
+_make_none_union('SomeOrderedDict',          'obj.dictdef')
 _make_none_union('SomeDict',          'obj.dictdef')
 _make_none_union('SomeWeakRef',         'obj.classdef')
 
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -5,9 +5,10 @@
 from __future__ import absolute_import
 
 import sys, types, inspect, weakref
+from collections import OrderedDict
 
 from rpython.flowspace.model import Constant
-from rpython.annotator.model import (
+from rpython.annotator.model import (SomeOrderedDict,
     SomeString, SomeChar, SomeFloat, SomePtr, unionof, SomeInstance, SomeDict,
     SomeBuiltin, SomePBC, SomeInteger, TLS, SomeAddress, SomeUnicodeCodePoint,
     s_None, s_ImpossibleValue, SomeLLADTMeth, SomeBool, SomeTuple,
@@ -370,7 +371,7 @@
                 for e in x:
                     listdef.generalize(self.immutablevalue(e, False))
                 result = SomeList(listdef)
-        elif tp is dict or tp is r_dict:
+        elif tp is dict or tp is r_dict or tp is OrderedDict:
             if need_const:
                 key = Constant(x)
                 try:
@@ -412,7 +413,10 @@
                     dictdef.generalize_key(self.immutablevalue(ek, False))
                     dictdef.generalize_value(self.immutablevalue(ev, False))
                     dictdef.seen_prebuilt_key(ek)
-                result = SomeDict(dictdef)
+                if tp is OrderedDict:
+                    result = SomeOrderedDict(dictdef)
+                else:
+                    result = SomeDict(dictdef)
         elif tp is weakref.ReferenceType:
             x1 = x()
             if x1 is None:
diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -2,11 +2,13 @@
 Built-in functions.
 """
 import sys
+from collections import OrderedDict
 
 from rpython.annotator.model import (
     SomeInteger, SomeObject, SomeChar, SomeBool, SomeString, SomeTuple, s_Bool,
     SomeUnicodeCodePoint, SomeAddress, SomeFloat, unionof, SomeUnicodeString,
     SomePBC, SomeInstance, SomeDict, SomeList, SomeWeakRef, SomeIterator,
+    SomeOrderedDict,
     SomeByteArray, annotation_to_lltype, lltype_to_annotation,
     ll_to_annotation, add_knowntypedata, s_ImpossibleValue,)
 from rpython.annotator.bookkeeper import getbookkeeper
@@ -357,6 +359,7 @@
 BUILTIN_ANALYZERS[rpython.rlib.rarithmetic.longlongmask] = rarith_longlongmask
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.instantiate] = robjmodel_instantiate
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_dict] = robjmodel_r_dict
+BUILTIN_ANALYZERS[OrderedDict] = lambda : SomeOrderedDict(getbookkeeper().getdictdef())
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.hlinvoke] = robjmodel_hlinvoke
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.keepalive_until_here] = robjmodel_keepalive_until_here
 BUILTIN_ANALYZERS[rpython.rtyper.lltypesystem.llmemory.cast_ptr_to_adr] = llmemory_cast_ptr_to_adr
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -32,6 +32,7 @@
 import inspect
 import weakref
 from types import BuiltinFunctionType, MethodType
+from collections import OrderedDict
 
 import rpython
 from rpython.tool import descriptor
@@ -355,6 +356,18 @@
         else:
             return '{...%s...}' % (len(const),)
 
+class SomeOrderedDict(SomeDict):
+    knowntype = OrderedDict
+
+    def method_copy(dct):
+        return SomeOrderedDict(dct.dictdef)
+
+    def method_update(dct1, dct2):
+        if s_None.contains(dct2):
+            return SomeImpossibleValue()
+        assert isinstance(dct2, SomeOrderedDict), "OrderedDict.update(dict) not allowed"
+        dct1.dictdef.union(dct2.dictdef)
+
 
 class SomeIterator(SomeObject):
     "Stands for an iterator returning objects from a given container."
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -194,10 +194,11 @@
         return True
 
 
-class DictRepr(AbstractDictRepr):
+class OrderedDictRepr(AbstractDictRepr):
 
     def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
-                 custom_eq_hash=None):
+                 custom_eq_hash=None, force_non_null=False):
+        assert not force_non_null
         self.rtyper = rtyper
         self.finalized = False
         self.DICT = lltype.GcForwardReference()
@@ -389,7 +390,7 @@
         v_res = hop.gendirectcall(target, *v_args)
         return self.recast_value(hop.llops, v_res)
 
-class __extend__(pairtype(DictRepr, rmodel.Repr)):
+class __extend__(pairtype(OrderedDictRepr, rmodel.Repr)):
 
     def rtype_getitem((r_dict, r_key), hop):
         v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
@@ -419,7 +420,7 @@
         hop.exception_is_here()
         return hop.gendirectcall(ll_dict_contains, v_dict, v_key)
 
-class __extend__(pairtype(DictRepr, DictRepr)):
+class __extend__(pairtype(OrderedDictRepr, OrderedDictRepr)):
     def convert_from_to((r_dict1, r_dict2), v, llops):
         # check that we don't convert from Dicts with
         # different key/value types
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -1,3 +1,5 @@
+from collections import OrderedDict
+
 from rpython.annotator import model as annmodel
 from rpython.flowspace.model import Constant
 from rpython.rlib import rarithmetic, objectmodel
@@ -726,10 +728,19 @@
 
     raise TyperError("hasattr is only suported on a constant")
 
+def rtype_ordered_dict(hop):
+    from rpython.rtyper.lltypesystem.rordereddict import ll_newdict
+
+    hop.exception_cannot_occur()
+    r_dict = hop.r_result
+    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
+    return hop.gendirectcall(ll_newdict, cDICT)
+
 BUILTIN_TYPER[objectmodel.instantiate] = rtype_instantiate
 BUILTIN_TYPER[isinstance] = rtype_builtin_isinstance
 BUILTIN_TYPER[hasattr] = rtype_builtin_hasattr
 BUILTIN_TYPER[objectmodel.r_dict] = rtype_r_dict
+BUILTIN_TYPER[OrderedDict] = rtype_ordered_dict
 
 # _________________________________________________________________
 # weakrefs
diff --git a/rpython/rtyper/rdict.py b/rpython/rtyper/rdict.py
--- a/rpython/rtyper/rdict.py
+++ b/rpython/rtyper/rdict.py
@@ -4,8 +4,12 @@
 
 
 class __extend__(annmodel.SomeDict):
+    def get_dict_repr(self):
+        from rpython.rtyper.lltypesystem.rdict import DictRepr
+
+        return DictRepr
+
     def rtyper_makerepr(self, rtyper):
-        from rpython.rtyper.lltypesystem.rdict import DictRepr
         dictkey = self.dictdef.dictkey
         dictvalue = self.dictdef.dictvalue
         s_key = dictkey.s_value
@@ -16,7 +20,7 @@
                                       rtyper.getrepr(dictkey.s_rdict_hashfn))
         else:
             custom_eq_hash = None
-        return DictRepr(rtyper, lambda: rtyper.getrepr(s_key),
+        return self.get_dict_repr()(rtyper, lambda: rtyper.getrepr(s_key),
                         lambda: rtyper.getrepr(s_value), dictkey, dictvalue,
                         custom_eq_hash, force_non_null)
 
@@ -25,6 +29,11 @@
         self.dictdef.dictvalue.dont_change_any_more = True
         return (self.__class__, self.dictdef.dictkey, self.dictdef.dictvalue)
 
+class __extend__(annmodel.SomeOrderedDict):
+    def get_dict_repr(self):
+        from rpython.rtyper.lltypesystem.rordereddict import OrderedDictRepr
+
+        return OrderedDictRepr
 
 class AbstractDictRepr(rmodel.Repr):
 
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -22,11 +22,11 @@
         yield x
 
 
-class TestRdict(BaseRtypingTest):
-
+class BaseTestRDict(BaseRtypingTest):
     def test_dict_creation(self):
         def createdict(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             return d['hello']
 
         res = self.interpret(createdict, [42])
@@ -34,7 +34,8 @@
 
     def test_dict_getitem_setitem(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             d['world'] = i + 1
             return d['hello'] * d['world']
         res = self.interpret(func, [6])
@@ -42,7 +43,8 @@
 
     def test_dict_getitem_keyerror(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             try:
                 return d['world']
             except KeyError:
@@ -52,7 +54,8 @@
 
     def test_dict_del_simple(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             d['world'] = i + 1
             del d['hello']
             return len(d)
@@ -61,7 +64,8 @@
 
     def test_dict_clear(self):
         def func(i):
-            d = {'abc': i}
+            d = self.newdict()
+            d['abc'] = i
             d['def'] = i+1
             d.clear()
             d['ghi'] = i+2
@@ -72,7 +76,8 @@
 
     def test_empty_strings(self):
         def func(i):
-            d = {'' : i}
+            d = self.newdict()
+            d[''] = i
             del d['']
             try:
                 d['']
@@ -84,7 +89,8 @@
         assert res == 1
 
         def func(i):
-            d = {'' : i}
+            d = self.newdict()
+            d[''] = i
             del d['']
             d[''] = i + 1
             return len(d)
@@ -94,9 +100,10 @@
     def test_dict_bool(self):
         def func(i):
             if i:
-                d = {}
+                d = self.newdict()
             else:
-                d = {i: i+1}
+                d = self.newdict()
+                d[i] = i+1
             if d:
                 return i
             else:
@@ -106,17 +113,20 @@
 
     def test_contains(self):
         def func(x, y):
-            d = {x: x+1}
+            d = self.newdict()
+            d[x] = x+1
             return y in d
         assert self.interpret(func, [42, 0]) == False
         assert self.interpret(func, [42, 42]) == True
 
     def test_contains_2(self):
-        d = {'5': None, '7': None}
+        d = self.newdict()
+        d['5'] = None
+        d['7'] = None
         def func(x):
             return chr(x) in d
-        #assert self.interpret(func, [ord('5')]) == True
-        #assert self.interpret(func, [ord('6')]) == False
+        assert self.interpret(func, [ord('5')]) == True
+        assert self.interpret(func, [ord('6')]) == False
 
         def func(n):
             return str(n) in d
@@ -124,7 +134,7 @@
 
     def test_dict_iteration(self):
         def func(i, j):
-            d = {}
+            d = self.newdict()
             d['hello'] = i
             d['world'] = j
             k = 1
@@ -136,7 +146,7 @@
 
     def test_dict_itermethods(self):
         def func():
-            d = {}
+            d = self.newdict()
             d['hello'] = 6
             d['world'] = 7
             k1 = k2 = k3 = 1
@@ -151,19 +161,9 @@
         res = self.interpret(func, [])
         assert res == 42 + 42 + 42
 
-    def test_two_dicts_with_different_value_types(self):
-        def func(i):
-            d1 = {}
-            d1['hello'] = i + 1
-            d2 = {}
-            d2['world'] = d1
-            return d2['world']['hello']
-        res = self.interpret(func, [5])
-        assert res == 6
-
     def test_dict_get(self):
         def func():
-            dic = {}
+            dic = self.newdict()
             x1 = dic.get('hi', 42)
             dic['blah'] = 1 # XXX this triggers type determination
             x2 = dic.get('blah', 2)
@@ -174,7 +174,7 @@
     def test_dict_get_empty(self):
         def func():
             # this time without writing to the dict
-            dic = {}
+            dic = self.newdict()
             x1 = dic.get('hi', 42)
             x2 = dic.get('blah', 2)
             return x1 * 10 + x2
@@ -183,14 +183,14 @@
 
     def test_dict_setdefault(self):
         def f():
-            d = {}
+            d = self.newdict()
             d.setdefault('a', 2)
             return d['a']
         res = self.interpret(f, ())
         assert res == 2
 
         def f():
-            d = {}
+            d = self.newdict()
             d.setdefault('a', 2)
             x = d.setdefault('a', -3)
             return x
@@ -200,7 +200,9 @@
     def test_dict_copy(self):
         def func():
             # XXX this does not work if we use chars, only!
-            dic = {'ab':1, 'b':2}
+            dic = self.newdict()
+            dic['ab'] = 1
+            dic['b'] = 2
             d2 = dic.copy()
             ok = 1
             for key in d2:
@@ -215,8 +217,12 @@
 
     def test_dict_update(self):
         def func():
-            dic = {'ab':1000, 'b':200}
-            d2 = {'b':30, 'cb':4}
+            dic = self.newdict()
+            dic['ab'] = 1000
+            dic['b'] = 200
+            d2 = self.newdict()
+            d2['b'] = 30
+            d2['cb'] = 4
             dic.update(d2)
             ok = len(dic) == 3
             sum = ok
@@ -228,7 +234,9 @@
 
     def test_dict_keys(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             keys = dic.keys()
             return ord(keys[0][1]) + ord(keys[1][1]) - 2*ord('0') + len(keys)
         res = self.interpret(func, ())#, view=True)
@@ -240,8 +248,11 @@
         class A(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {A(): 1, A(): 2}
+            dic0 = self.newdict()
+            dic0[Empty()] = 2
+            dic = self.newdict()
+            dic[A()] = 1
+            dic[A()] = 2
             keys = dic.keys()
             return (isinstance(keys[1], A))*2+(isinstance(keys[0],A))
         res = self.interpret(func, [])
@@ -253,8 +264,11 @@
         class A(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {A(): 1, A(): 2}
+            dic0 = self.newdict()
+            dic0[Empty()] = 2
+            dic = self.newdict()
+            dic[A()] = 1
+            dic[A()] = 2
             a = 0
             for k in dic.iterkeys():
                 a += isinstance(k, A)
@@ -264,7 +278,9 @@
 
     def test_dict_values(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             values = dic.values()
             return values[0] + values[1] + len(values)
         res = self.interpret(func, ())
@@ -274,7 +290,9 @@
         class A:
             pass
         def func():
-            dic = {1: A(), 2: A()}
+            dic = self.newdict()
+            dic[1] = A()
+            dic[2] = A()
             vals = dic.values()
             return (isinstance(vals[1], A))*2+(isinstance(vals[0],A))
         res = self.interpret(func, [])
@@ -284,7 +302,9 @@
         class A:
             pass
         def func():
-            dic = {1: A(), 2: A()}
+            dic = self.newdict()
+            dic[1] = A()
+            dic[2] = A()
             a = 0
             for v in dic.itervalues():
                 a += isinstance(v, A)
@@ -300,8 +320,11 @@
         class B(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {B(): A(), B(): A()}
+            dic0 = self.newdict()
+            dic0[Empty()] = A()
+            dic = self.newdict()
+            dic[B()] = A()
+            dic[B()] = A()
             items = dic.items()
             b = 0
             a = 0
@@ -320,8 +343,11 @@
         class B(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {B(): A(), B(): A()}
+            dic0 = self.newdict()
+            dic0[Empty()] = A()
+            dic = self.newdict()
+            dic[B()] = A()
+            dic[B()] = A()
             b = 0
             a = 0
             for k, v in dic.iteritems():
@@ -333,7 +359,9 @@
 
     def test_dict_items(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             items = dic.items()
             res = len(items)
             for key, value in items:
@@ -344,13 +372,17 @@
 
     def test_dict_contains(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             return ' 4' in dic and ' 9' not in dic
         res = self.interpret(func, ())
         assert res is True
 
     def test_dict_contains_with_constant_dict(self):
-        dic = {'4':1000, ' 8':200}
+        dic = self.newdict()
+        dic['4'] = 1000
+        dic['8'] = 200
         def func(i):
             return chr(i) in dic
         res = self.interpret(func, [ord('4')])
@@ -367,7 +399,9 @@
             a = A()
             a.d = None
             if n > 0:
-                a.d = {str(n): 1, "42": 2}
+                a.d = self.newdict()
+                a.d[str(n)] = 1
+                a.d["42"] = 2
                 del a.d["42"]
             return negate(a.d)
         res = self.interpret(func, [10])
@@ -379,7 +413,8 @@
 
     def test_int_dict(self):
         def func(a, b):
-            dic = {12: 34}
+            dic = self.newdict()
+            dic[12] = 34
             dic[a] = 1000
             return dic.get(b, -123)
         res = self.interpret(func, [12, 12])
@@ -403,7 +438,7 @@
         def f():
             a = A()
             b = B()
-            d = {}
+            d = self.newdict()
             d[b] = 7
             d[a] = 3
             return len(d) + d[a] + d[b]
@@ -411,7 +446,9 @@
         assert res == 12
 
     def test_captured_get(self):
-        get = {1:2}.get
+        d = self.newdict()
+        d[1] = 2
+        get = d.get
         def f():
             return get(1, 3)+get(2, 4)
         res = self.interpret(f, [])
@@ -431,40 +468,21 @@
         def f():
             lst = [A()]
             res1 = A() in lst
-            d2 = {B(): None, B(): None}
+            d2 = self.newdict()
+            d2[B()] = None
+            d2[B()] = None
             return res1+len(d2)
         res = self.interpret(f, [])
         assert res == 2
 
-
-    def test_type_erase(self):
-        class A(object):
-            pass
-        class B(object):
-            pass
-
-        def f():
-            return {A(): B()}, {B(): A()}
-
-        t = TranslationContext()
-        s = t.buildannotator().build_types(f, [])
-        rtyper = t.buildrtyper()
-        rtyper.specialize()
-
-        s_AB_dic = s.items[0]
-        s_BA_dic = s.items[1]
-
-        r_AB_dic = rtyper.getrepr(s_AB_dic)
-        r_BA_dic = rtyper.getrepr(s_AB_dic)
-
-        assert r_AB_dic.lowleveltype == r_BA_dic.lowleveltype
-
     def test_identity_hash_is_fast(self):
         class A(object):
             pass
 
         def f():
-            return {A(): 1}
+            d = self.newdict()
+            d[A()] = 1
+            return d
 
         t = TranslationContext()
         s = t.buildannotator().build_types(f, [])
@@ -476,7 +494,7 @@
 
     def test_tuple_dict(self):
         def f(i):
-            d = {}
+            d = self.newdict()
             d[(1, 4.5, (str(i), 2), 2)] = 4
             d[(1, 4.5, (str(i), 2), 3)] = 6
             return d[(1, 4.5, (str(i), 2), i)]
@@ -486,9 +504,9 @@
 
     def test_dict_of_dict(self):
         def f(n):
-            d = {}
+            d = self.newdict()
             d[5] = d
-            d[6] = {}
+            d[6] = self.newdict()
             return len(d[n])
 
         res = self.interpret(f, [5])
@@ -504,10 +522,9 @@
             pass
 
         def f(i):
-            d = {
-                A: 3,
-                B: 4,
-            }
+            d = self.newdict()
+            d[A] = 3
+            d[B] = 4
             if i:
                 cls = A
             else:
@@ -526,7 +543,9 @@
         class B(A):
             pass
 
-        d = {(A, 3): 3, (B, 0): 4}
+        d = self.newdict()
+        d[(A, 3)] = 3
+        d[(B, 0)] = 4
 
         def f(i):
             if i:
@@ -553,7 +572,9 @@
                 return 42
             return -1
         def g(n):
-            d = {1: n, 2: 2*n}
+            d = self.newdict()
+            d[1] = n
+            d[2] = 2*n
             return f(d)
         res = self.interpret(g, [3])
         assert res == 6
@@ -566,7 +587,8 @@
                 return 42
             return -1
         def g(n):
-            d = {1: n}
+            d = self.newdict()
+            d[1] = n
             f(d)
             return d[2]
         res = self.interpret(g, [3])
@@ -610,7 +632,10 @@
 
     def test_resize_during_iteration(self):
         def func():
-            d = {5: 1, 6: 2, 7: 3}
+            d = self.newdict()
+            d[5] = 1
+            d[6] = 2
+            d[7] = 3
             try:
                 for key, value in d.iteritems():
                     d[key^16] = value*2
@@ -625,14 +650,21 @@
 
     def test_change_during_iteration(self):
         def func():
-            d = {'a': 1, 'b': 2}
+            d = self.newdict()
+            d['a'] = 1
+            d['b'] = 2
             for key in d:
                 d[key] = 42
             return d['a']
         assert self.interpret(func, []) == 42
 
     def test_dict_of_floats(self):
-        d = {3.0: 42, 3.1: 43, 3.2: 44, 3.3: 45, 3.4: 46}
+        d = self.newdict()
+        d[3.0] = 42
+        d[3.1] = 43
+        d[3.2] = 44
+        d[3.3] = 45
+        d[3.4] = 46
         def fn(f):
             return d[f]
 
@@ -643,7 +675,9 @@
         for r_t in [r_uint, r_longlong, r_ulonglong]:
             if r_t is r_int:
                 continue    # for 64-bit platforms: skip r_longlong
-            d = {r_t(2): 3, r_t(4): 5}
+            d = self.newdict()
+            d[r_t(2)] = 3
+            d[r_t(4)] = 5
             def fn(x, y):
                 d[r_t(x)] = 123
                 return d[r_t(y)]
@@ -654,7 +688,7 @@
 
     def test_dict_popitem(self):
         def func():
-            d = {}
+            d = self.newdict()
             d[5] = 2
             d[6] = 3
             k1, v1 = d.popitem()
@@ -698,7 +732,7 @@
 
     def test_dict_pop(self):
         def f(n, default):
-            d = {}
+            d = self.newdict()
             d[2] = 3
             d[4] = 5
             if default == -1:
@@ -720,7 +754,7 @@
         class A(object):
             pass
         def f(n):
-            d = {}
+            d = self.newdict()
             d[2] = A()
             x = d.pop(n, None)
             if x is None:
@@ -734,7 +768,8 @@
 
     def test_dict_but_not_with_char_keys(self):
         def func(i):
-            d = {'h': i}
+            d = self.newdict()
+            d['h'] = i
             try:
                 return d['hello']
             except KeyError:
@@ -752,7 +787,7 @@
         def func(c1, c2):
             c1 = chr(c1)
             c2 = chr(c2)
-            d = {}
+            d = self.newdict()
             d[c1] = 1
             d[c2] = 2
             del d[c1]
@@ -806,27 +841,6 @@
                 count_frees += 1
         assert count_frees >= 3
 
-    def test_dict_resize(self):
-        # XXX we no longer automatically resize on 'del'.  We need to
-        # hack a bit in this test to trigger a resize by continuing to
-        # fill the dict's table while keeping the actual size very low
-        # in order to force a resize to shrink the table back
-        def func(want_empty):
-            d = {}
-            for i in range(rdict.DICT_INITSIZE << 1):
-                d[chr(ord('a') + i)] = i
-            if want_empty:
-                for i in range(rdict.DICT_INITSIZE << 1):
-                    del d[chr(ord('a') + i)]
-                for i in range(rdict.DICT_INITSIZE << 3):
-                    d[chr(ord('A') - i)] = i
-                    del d[chr(ord('A') - i)]
-            return d
-        res = self.interpret(func, [0])
-        assert len(res.entries) > rdict.DICT_INITSIZE
-        res = self.interpret(func, [1])
-        assert len(res.entries) == rdict.DICT_INITSIZE
-
     def test_dict_valid_resize(self):
         # see if we find our keys after resize
         def func():
@@ -843,6 +857,212 @@
 
     # ____________________________________________________________
 
+    def test_dict_of_addresses(self):
+        from rpython.rtyper.lltypesystem import llmemory
+        TP = lltype.Struct('x')
+        a = lltype.malloc(TP, flavor='raw', immortal=True)
+        b = lltype.malloc(TP, flavor='raw', immortal=True)
+
+        def func(i):
+            d = self.newdict()
+            d[llmemory.cast_ptr_to_adr(a)] = 123
+            d[llmemory.cast_ptr_to_adr(b)] = 456
+            if i > 5:
+                key = llmemory.cast_ptr_to_adr(a)
+            else:
+                key = llmemory.cast_ptr_to_adr(b)
+            return d[key]
+
+        assert self.interpret(func, [3]) == 456
+
+    def test_prebuilt_list_of_addresses(self):
+        from rpython.rtyper.lltypesystem import llmemory
+
+        TP = lltype.Struct('x', ('y', lltype.Signed))
+        a = lltype.malloc(TP, flavor='raw', immortal=True)
+        b = lltype.malloc(TP, flavor='raw', immortal=True)
+        c = lltype.malloc(TP, flavor='raw', immortal=True)
+        a_a = llmemory.cast_ptr_to_adr(a)
+        a0 = llmemory.cast_ptr_to_adr(a)
+        assert a_a is not a0
+        assert a_a == a0
+        a_b = llmemory.cast_ptr_to_adr(b)
+        a_c = llmemory.cast_ptr_to_adr(c)
+
+        d = self.newdict()
+        d[a_a] = 3
+        d[a_b] = 4
+        d[a_c] = 5
+        d[a0] = 8
+
+        def func(i):
+            if i == 0:
+                ptr = a
+            else:
+                ptr = b
+            return d[llmemory.cast_ptr_to_adr(ptr)]
+
+        py.test.raises(TypeError, self.interpret, func, [0])
+
+    def test_dict_of_voidp(self):
+        def func():
+            d = self.newdict()
+            handle = lltype.nullptr(rffi.VOIDP.TO)
+            # Use a negative key, so the dict implementation uses
+            # the value as a marker for empty entries
+            d[-1] = handle
+            return len(d)
+
+        assert self.interpret(func, []) == 1
+        from rpython.translator.c.test.test_genc import compile
+        f = compile(func, [])
+        res = f()
+        assert res == 1
+
+    def test_dict_with_SHORT_keys(self):
+        def func(x):
+            d = self.newdict()
+            d[rffi.cast(rffi.SHORT, 42)] = 123
+            d[rffi.cast(rffi.SHORT, -43)] = 321
+            return d[rffi.cast(rffi.SHORT, x)]
+
+        assert self.interpret(func, [42]) == 123
+        assert self.interpret(func, [2**16 - 43]) == 321
+
+    def test_dict_with_bool_keys(self):
+        def func(x):
+            d = self.newdict()
+            d[False] = 123
+            d[True] = 321
+            return d[x == 42]
+
+        assert self.interpret(func, [5]) == 123
+        assert self.interpret(func, [42]) == 321
+
+    def test_nonnull_hint(self):
+        def eq(a, b):
+            return a == b
+        def rhash(a):
+            return 3
+
+        def func(i):
+            d = r_dict(eq, rhash, force_non_null=True)
+            if not i:
+                d[None] = i
+            else:
+                d[str(i)] = i
+            return "12" in d, d
+
+        llres = self.interpret(func, [12])
+        assert llres.item0 == 1
+        DICT = lltype.typeOf(llres.item1)
+        assert sorted(DICT.TO.entries.TO.OF._flds) == ['f_hash', 'key', 'value']
+
+    def test_memoryerror_should_not_insert(self):
+        # This shows a misbehaviour that also exists in CPython 2.7, but not
+        # any more in CPython 3.3.  The behaviour is that even if a dict
+        # insertion raises MemoryError, the new item is still inserted.
+        # If we catch the MemoryError, we can keep inserting new items until
+        # the dict table is completely full.  Then the next insertion loops
+        # forever.  This test only checks that after a MemoryError the
+        # new item was not inserted.
+        def _check_small_range(self, n):
+            if n >= 128:
+                raise MemoryError
+            return range(n)
+        original_check_range = lltype._array._check_range
+        try:
+            lltype._array._check_range = _check_small_range
+            #
+            def do_insert(d, i):
+                d[i] = i
+            def func():
+                d = self.newdict()
+                i = 0
+                while True:
+                    try:
+                        do_insert(d, i)
+                    except MemoryError:
+                        return (i in d)
+                    i += 1
+            res = self.interpret(func, [])
+            assert res == 0
+            #
+        finally:
+            lltype._array._check_range = original_check_range
+
+    def test_dict_with_none_key(self):
+        def func(i):
+            d = self.newdict()
+            d[None] = i
+            return d[None]
+        res = self.interpret(func, [42])
+        assert res == 42
+
+
+class TestRDict(BaseTestRDict):
+    @staticmethod
+    def newdict():
+        return {}
+
+    def test_two_dicts_with_different_value_types(self):
+        def func(i):
+            d1 = {}
+            d1['hello'] = i + 1
+            d2 = {}
+            d2['world'] = d1
+            return d2['world']['hello']
+        res = self.interpret(func, [5])
+        assert res == 6
+
+    def test_type_erase(self):
+        class A(object):
+            pass
+        class B(object):
+            pass
+
+        def f():
+            d = {}
+            d[A()] = B()
+            d2 = {}
+            d2[B()] = A()
+            return d, d2
+
+        t = TranslationContext()
+        s = t.buildannotator().build_types(f, [])
+        rtyper = t.buildrtyper()
+        rtyper.specialize()
+
+        s_AB_dic = s.items[0]
+        s_BA_dic = s.items[1]
+
+        r_AB_dic = rtyper.getrepr(s_AB_dic)
+        r_BA_dic = rtyper.getrepr(s_AB_dic)
+
+        assert r_AB_dic.lowleveltype == r_BA_dic.lowleveltype
+
+
+    def test_dict_resize(self):
+        # XXX we no longer automatically resize on 'del'.  We need to
+        # hack a bit in this test to trigger a resize by continuing to
+        # fill the dict's table while keeping the actual size very low
+        # in order to force a resize to shrink the table back
+        def func(want_empty):
+            d = self.newdict()
+            for i in range(rdict.DICT_INITSIZE << 1):
+                d[chr(ord('a') + i)] = i
+            if want_empty:
+                for i in range(rdict.DICT_INITSIZE << 1):
+                    del d[chr(ord('a') + i)]
+                for i in range(rdict.DICT_INITSIZE << 3):
+                    d[chr(ord('A') - i)] = i
+                    del d[chr(ord('A') - i)]
+            return d
+        res = self.interpret(func, [0])
+        assert len(res.entries) > rdict.DICT_INITSIZE
+        res = self.interpret(func, [1])
+        assert len(res.entries) == rdict.DICT_INITSIZE
+
     def test_opt_nullkeymarker(self):
         def f():
             d = {"hello": None}
@@ -940,145 +1160,6 @@
         assert lltype.typeOf(res.item1) == lltype.typeOf(res.item2)
         assert lltype.typeOf(res.item1) == lltype.typeOf(res.item3)
 
-    def test_dict_of_addresses(self):
-        from rpython.rtyper.lltypesystem import llmemory
-        TP = lltype.Struct('x')
-        a = lltype.malloc(TP, flavor='raw', immortal=True)
-        b = lltype.malloc(TP, flavor='raw', immortal=True)
-
-        def func(i):
-            d = {}
-            d[llmemory.cast_ptr_to_adr(a)] = 123
-            d[llmemory.cast_ptr_to_adr(b)] = 456
-            if i > 5:
-                key = llmemory.cast_ptr_to_adr(a)
-            else:
-                key = llmemory.cast_ptr_to_adr(b)
-            return d[key]
-
-        assert self.interpret(func, [3]) == 456
-
-    def test_prebuilt_list_of_addresses(self):
-        from rpython.rtyper.lltypesystem import llmemory
-
-        TP = lltype.Struct('x', ('y', lltype.Signed))
-        a = lltype.malloc(TP, flavor='raw', immortal=True)
-        b = lltype.malloc(TP, flavor='raw', immortal=True)
-        c = lltype.malloc(TP, flavor='raw', immortal=True)
-        a_a = llmemory.cast_ptr_to_adr(a)
-        a0 = llmemory.cast_ptr_to_adr(a)
-        assert a_a is not a0
-        assert a_a == a0
-        a_b = llmemory.cast_ptr_to_adr(b)
-        a_c = llmemory.cast_ptr_to_adr(c)
-
-        d = {a_a: 3, a_b: 4, a_c: 5}
-        d[a0] = 8
-
-        def func(i):
-            if i == 0:
-                ptr = a
-            else:
-                ptr = b
-            return d[llmemory.cast_ptr_to_adr(ptr)]
-
-        py.test.raises(TypeError, self.interpret, func, [0])
-
-    def test_dict_of_voidp(self):
-        def func():
-            d = {}
-            handle = lltype.nullptr(rffi.VOIDP.TO)
-            # Use a negative key, so the dict implementation uses
-            # the value as a marker for empty entries
-            d[-1] = handle
-            return len(d)
-
-        assert self.interpret(func, []) == 1
-        from rpython.translator.c.test.test_genc import compile
-        f = compile(func, [])
-        res = f()
-        assert res == 1
-
-    def test_dict_with_SHORT_keys(self):
-        def func(x):
-            d = {}
-            d[rffi.cast(rffi.SHORT, 42)] = 123
-            d[rffi.cast(rffi.SHORT, -43)] = 321
-            return d[rffi.cast(rffi.SHORT, x)]
-
-        assert self.interpret(func, [42]) == 123
-        assert self.interpret(func, [2**16 - 43]) == 321
-
-    def test_dict_with_bool_keys(self):
-        def func(x):
-            d = {}
-            d[False] = 123
-            d[True] = 321
-            return d[x == 42]
-
-        assert self.interpret(func, [5]) == 123
-        assert self.interpret(func, [42]) == 321
-
-    def test_nonnull_hint(self):
-        def eq(a, b):
-            return a == b
-        def rhash(a):
-            return 3
-
-        def func(i):
-            d = r_dict(eq, rhash, force_non_null=True)
-            if not i:
-                d[None] = i
-            else:
-                d[str(i)] = i
-            return "12" in d, d
-
-        llres = self.interpret(func, [12])
-        assert llres.item0 == 1
-        DICT = lltype.typeOf(llres.item1)
-        assert sorted(DICT.TO.entries.TO.OF._flds) == ['f_hash', 'key', 'value']
-
-    def test_memoryerror_should_not_insert(self):
-        # This shows a misbehaviour that also exists in CPython 2.7, but not
-        # any more in CPython 3.3.  The behaviour is that even if a dict
-        # insertion raises MemoryError, the new item is still inserted.
-        # If we catch the MemoryError, we can keep inserting new items until
-        # the dict table is completely full.  Then the next insertion loops
-        # forever.  This test only checks that after a MemoryError the
-        # new item was not inserted.
-        def _check_small_range(self, n):
-            if n >= 128:
-                raise MemoryError
-            return range(n)
-        original_check_range = lltype._array._check_range
-        try:
-            lltype._array._check_range = _check_small_range
-            #
-            def do_insert(d, i):
-                d[i] = i
-            def func():
-                d = {}
-                i = 0
-                while True:
-                    try:
-                        do_insert(d, i)
-                    except MemoryError:
-                        return (i in d)
-                    i += 1
-            res = self.interpret(func, [])
-            assert res == 0
-            #
-        finally:
-            lltype._array._check_range = original_check_range
-
-    def test_dict_with_none_key(self):
-        def func(i):
-            d = {None: i}
-            return d[None]
-        res = self.interpret(func, [42])
-        assert res == 42
-
-
 class TestStress:
 
     def test_stress(self):
@@ -1193,3 +1274,4 @@
                 print 'current dict length:', referencelength
             assert l_dict.num_items == referencelength
         complete_check()
+
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -1,9 +1,11 @@
 
 import py
+from collections import OrderedDict
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem import rordereddict, rstr
 from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.annlowlevel import llstr, hlstr
+from rpython.rtyper.test.test_rdict import BaseTestRDict
 
 
 def get_indexes(ll_d):
@@ -21,7 +23,7 @@
             c += 1
     return c
 
-    
+
 class TestRDictDirect(object):
     dummykeyobj = None
     dummyvalueobj = None
@@ -252,3 +254,24 @@
 class TestRDictDirectDummyValue(TestRDictDirect):
     class dummyvalueobj:
         ll_dummy_value = -42
+
+class TestOrderedRDict(BaseTestRDict):
+    @staticmethod
+    def newdict():
+        return OrderedDict()
+
+    def test_two_dicts_with_different_value_types(self):
+        def func(i):
+            d1 = OrderedDict()
+            d1['hello'] = i + 1
+            d2 = OrderedDict()
+            d2['world'] = d1
+            return d2['world']['hello']
+        res = self.interpret(func, [5])
+        assert res == 6
+
+    def test_dict_with_SHORT_keys(self):
+        py.test.skip("I don't want to edit this file on two branches")
+
+    def test_memoryerror_should_not_insert(self):
+        py.test.skip("I don't want to edit this file on two branches")

From noreply at buildbot.pypy.org  Tue Oct 29 00:40:41 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue, 29 Oct 2013 00:40:41 +0100 (CET)
Subject: [pypy-commit] pypy py3k: kill the need for cStringIO on py3,
 skip list strategy optimizations for now
Message-ID: <20131028234041.143421C00EC@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r67670:13f682c3d74d
Date: 2013-10-28 15:31 -0700
http://bitbucket.org/pypy/pypy/changeset/13f682c3d74d/

Log:	kill the need for cStringIO on py3, skip list strategy optimizations
	for now

diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -4,7 +4,7 @@
 
 
 class AppTest_fast_path_from_list(object):
-    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+    spaceconfig = dict(usemodules=('_cffi_backend',))
 
     def setup_method(self, meth):
         def forbidden(*args):
@@ -16,6 +16,7 @@
         W_CType.pack_list_of_items = self._original
 
     def test_fast_init_from_list(self):
+        py3k_skip('XXX: strategies are currently broken')
         import _cffi_backend
         LONG = _cffi_backend.new_primitive_type('long')
         P_LONG = _cffi_backend.new_pointer_type(LONG)
@@ -36,6 +37,7 @@
         assert buf[2] == 3.3
 
     def test_fast_init_short_from_list(self):
+        py3k_skip('XXX: strategies are currently broken')
         import _cffi_backend
         SHORT = _cffi_backend.new_primitive_type('short')
         P_SHORT = _cffi_backend.new_pointer_type(SHORT)
@@ -48,6 +50,7 @@
         raises(OverflowError, _cffi_backend.newp, SHORT_ARRAY, [-40000])
 
     def test_fast_init_longlong_from_list(self):
+        py3k_skip('XXX: strategies are currently broken')
         if type(2 ** 50) is long:
             large_int = 2 ** 30
         else:
@@ -63,6 +66,7 @@
         assert buf[3] == large_int
 
     def test_fast_init_ushort_from_list(self):
+        py3k_skip('XXX: strategies are currently broken')
         import _cffi_backend
         USHORT = _cffi_backend.new_primitive_type('unsigned short')
         P_USHORT = _cffi_backend.new_pointer_type(USHORT)
@@ -75,6 +79,7 @@
         raises(OverflowError, _cffi_backend.newp, USHORT_ARRAY, [-1])
 
     def test_fast_init_ulong_from_list(self):
+        py3k_skip('XXX: strategies are currently broken')
         import sys
         import _cffi_backend
         ULONG = _cffi_backend.new_primitive_type('unsigned long')
@@ -106,6 +111,7 @@
         assert float(buf[1]) == -3.5
 
     def test_fast_init_bool_from_list(self):
+        py3k_skip('XXX: strategies are currently broken')
         import _cffi_backend
         BOOL = _cffi_backend.new_primitive_type('_Bool')
         P_BOOL = _cffi_backend.new_pointer_type(BOOL)
@@ -119,7 +125,7 @@
 
 
 class AppTest_fast_path_bug(object):
-    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+    spaceconfig = dict(usemodules=('_cffi_backend',))
 
     def test_bug_not_list_or_tuple(self):
         import _cffi_backend
@@ -132,7 +138,7 @@
 
 
 class AppTest_fast_path_to_list(object):
-    spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO'))
+    spaceconfig = dict(usemodules=('_cffi_backend',))
 
     def setup_method(self, meth):
         from pypy.interpreter import gateway

From noreply at buildbot.pypy.org  Tue Oct 29 02:17:00 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 02:17:00 +0100 (CET)
Subject: [pypy-commit] pypy default: provide ndarray.{take, ptp}
Message-ID: <20131029011700.6E0EF1C0163@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67671:95d235f14315
Date: 2013-10-28 20:53 -0400
http://bitbucket.org/pypy/pypy/changeset/95d235f14315/

Log:	provide ndarray.{take, ptp}

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import operationerrfmt, OperationError
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import interp2app, unwrap_spec, applevel, \
+                                     WrappedDefault
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array,\
      ArrayArgumentException, issequence_w, wrap_impl
 from pypy.module.micronumpy import interp_dtype, interp_ufuncs, interp_boxes,\
@@ -359,18 +360,11 @@
                 "order not implemented"))
         return self.descr_reshape(space, [space.wrap(-1)])
 
-    def descr_take(self, space, w_obj, w_axis=None, w_out=None):
-        # if w_axis is None and w_out is Nont this is an equivalent to
-        # fancy indexing
-        raise OperationError(space.w_NotImplementedError,
-                             space.wrap("unsupported for now"))
-        if not space.is_none(w_axis):
-            raise OperationError(space.w_NotImplementedError,
-                                 space.wrap("axis unsupported for take"))
-        if not space.is_none(w_out):
-            raise OperationError(space.w_NotImplementedError,
-                                 space.wrap("out unsupported for take"))
-        return self.getitem_int(space, convert_to_array(space, w_obj))
+    @unwrap_spec(w_axis=WrappedDefault(None),
+                 w_out=WrappedDefault(None),
+                 w_mode=WrappedDefault('raise'))
+    def descr_take(self, space, w_obj, w_axis, w_out, w_mode):
+        return app_take(space, self, w_obj, w_axis, w_out, w_mode)
 
     def descr_compress(self, space, w_obj, w_axis=None):
         if not space.is_none(w_axis):
@@ -581,9 +575,10 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "newbyteorder not implemented yet"))
 
-    def descr_ptp(self, space, w_axis=None, w_out=None):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "ptp (peak to peak) not implemented yet"))
+    @unwrap_spec(w_axis=WrappedDefault(None),
+                 w_out=WrappedDefault(None))
+    def descr_ptp(self, space, w_axis, w_out):
+        return app_ptp(space, self, w_axis, w_out)
 
     def descr_put(self, space, w_indices, w_values, w_mode=None):
         put(space, self, w_indices, w_values, w_mode)
@@ -1005,6 +1000,32 @@
     else:
         return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype)
 
+app_take = applevel(r"""
+    def take(a, indices, axis, out, mode):
+        assert mode == 'raise'
+        if axis is None:
+            res = a.ravel()[indices]
+        else:
+            if axis < 0: axis += len(a.shape)
+            s0, s1 = a.shape[:axis], a.shape[axis+1:]
+            l0 = prod(s0) if s0 else 1
+            l1 = prod(s1) if s1 else 1
+            res = a.reshape((l0, -1, l1))[:,indices,:].reshape(s0 + (-1,) + s1)
+        if out is not None:
+            out[:] = res
+            return out
+        return res
+""", filename=__file__).interphook('take')
+
+app_ptp = applevel(r"""
+    def ptp(a, axis, out):
+        res = a.max(axis) - a.min(axis)
+        if out is not None:
+            out[:] = res
+            return out
+        return res
+""", filename=__file__).interphook('ptp')
+
 W_NDimArray.typedef = TypeDef(
     "ndarray",
     __module__ = "numpypy",
@@ -1114,6 +1135,7 @@
     flatten = interp2app(W_NDimArray.descr_flatten),
     ravel = interp2app(W_NDimArray.descr_ravel),
     take = interp2app(W_NDimArray.descr_take),
+    ptp = interp2app(W_NDimArray.descr_ptp),
     compress = interp2app(W_NDimArray.descr_compress),
     repeat = interp2app(W_NDimArray.descr_repeat),
     swapaxes = interp2app(W_NDimArray.descr_swapaxes),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2450,10 +2450,6 @@
 
     def test_take(self):
         from numpypy import arange
-        try:
-            arange(10).take([0])
-        except NotImplementedError:
-            skip("we wait for int-based indexing")
         assert (arange(10).take([1, 2, 1, 1]) == [1, 2, 1, 1]).all()
         raises(IndexError, "arange(3).take([15])")
         a = arange(6).reshape(2, 3)
@@ -2462,6 +2458,13 @@
         a = arange(12).reshape(2, 6)
         assert (a[:,::2].take([3, 2, 1]) == [6, 4, 2]).all()
 
+    def test_ptp(self):
+        import numpypy as np
+        x = np.arange(4).reshape((2,2))
+        assert x.ptp() == 3
+        assert (x.ptp(axis=0) == [2, 2]).all()
+        assert (x.ptp(axis=1) == [1, 1]).all()
+
     def test_compress(self):
         from numpypy import arange, array
         a = arange(10)

From noreply at buildbot.pypy.org  Tue Oct 29 04:36:17 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 04:36:17 +0100 (CET)
Subject: [pypy-commit] pypy default: fix translation after last commit
Message-ID: <20131029033617.125151C13DA@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67672:57fb97ec48c9
Date: 2013-10-28 23:35 -0400
http://bitbucket.org/pypy/pypy/changeset/57fb97ec48c9/

Log:	fix translation after last commit

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -363,7 +363,7 @@
     @unwrap_spec(w_axis=WrappedDefault(None),
                  w_out=WrappedDefault(None),
                  w_mode=WrappedDefault('raise'))
-    def descr_take(self, space, w_obj, w_axis, w_out, w_mode):
+    def descr_take(self, space, w_obj, w_axis=None, w_out=None, w_mode=None):
         return app_take(space, self, w_obj, w_axis, w_out, w_mode)
 
     def descr_compress(self, space, w_obj, w_axis=None):
@@ -577,7 +577,7 @@
 
     @unwrap_spec(w_axis=WrappedDefault(None),
                  w_out=WrappedDefault(None))
-    def descr_ptp(self, space, w_axis, w_out):
+    def descr_ptp(self, space, w_axis=None, w_out=None):
         return app_ptp(space, self, w_axis, w_out)
 
     def descr_put(self, space, w_indices, w_values, w_mode=None):

From noreply at buildbot.pypy.org  Tue Oct 29 05:35:22 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 05:35:22 +0100 (CET)
Subject: [pypy-commit] pypy default: fix segfault when accessing real/imag
 views of nonnative complex types
Message-ID: <20131029043522.7D3FF1C1054@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67673:0e1c4dd7f78d
Date: 2013-10-28 23:23 -0400
http://bitbucket.org/pypy/pypy/changeset/0e1c4dd7f78d/

Log:	fix segfault when accessing real/imag views of nonnative complex
	types

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -50,7 +50,7 @@
     return out
 
 class W_Dtype(W_Root):
-    _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder"]
+    _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder", "float_type"]
 
     def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder='=',
                  alternate_constructors=[], aliases=[], float_type=None,
@@ -749,7 +749,7 @@
             self.dtypes_by_name[new_name] = W_Dtype(
                 itemtype,
                 dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                byteorder=OPPBYTE)
+                byteorder=OPPBYTE, float_type=dtype.float_type)
             if dtype.kind != dtype.char:
                 can_name = dtype.char
                 self.dtypes_by_name[NATBYTE + can_name] = dtype
@@ -758,7 +758,7 @@
                 self.dtypes_by_name[new_name] = W_Dtype(
                     itemtype,
                     dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                    byteorder=OPPBYTE)
+                    byteorder=OPPBYTE, float_type=dtype.float_type)
 
             for alias in dtype.aliases:
                 self.dtypes_by_name[alias] = dtype
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1466,6 +1466,11 @@
         assert a[3].imag == -10
         assert a[2].imag == -5
 
+        assert arange(4, dtype='>c8').imag.max() == 0.0
+        assert arange(4, dtype='<c8').imag.max() == 0.0
+        assert arange(4, dtype='>c8').real.max() == 3.0
+        assert arange(4, dtype='<c8').real.max() == 3.0
+
     def test_trace(self):
         import numpypy as np
         assert np.trace(np.eye(3)) == 3.0

From noreply at buildbot.pypy.org  Tue Oct 29 06:43:45 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 06:43:45 +0100 (CET)
Subject: [pypy-commit] pypy default: random cleanups
Message-ID: <20131029054345.60CA91C0163@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67674:e0f88f35c8e3
Date: 2013-10-29 01:09 -0400
http://bitbucket.org/pypy/pypy/changeset/e0f88f35c8e3/

Log:	random cleanups

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -680,8 +680,8 @@
         # strange
         assert dtype('string').str == '|S0'
         assert dtype('unicode').str == byteorder + 'U0'
-        # assert dtype(('string', 7)).str == '|S7'
-        # assert dtype(('unicode', 7)).str == '<U7'
+        assert dtype(('string', 7)).str == '|S7'
+        assert dtype(('unicode', 7)).str == '<U7'
 
     def test_intp(self):
         from numpypy import dtype
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1648,8 +1648,6 @@
                                             dtype=[('x', int), ('y', float)])))
         assert str(exc.value).startswith('invalid type promotion')
 
-
-
     def test_std(self):
         from numpypy import array
         a = array(range(10))
@@ -1966,9 +1964,7 @@
         b = np.array([True])
 
         assert (a[b] == a).all()
-
         a[b] = 1.
-
         assert (a == [[1., 1., 1.]]).all()
 
     @py.test.mark.xfail

From noreply at buildbot.pypy.org  Tue Oct 29 09:35:42 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 09:35:42 +0100 (CET)
Subject: [pypy-commit] pypy default: ndarray{argmin, argmax,
 transpose} need to accept extra args, even if unsupported
Message-ID: <20131029083542.2388E1C0330@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67675:a009f51da9e4
Date: 2013-10-29 04:31 -0400
http://bitbucket.org/pypy/pypy/changeset/a009f51da9e4/

Log:	ndarray{argmin,argmax,transpose} need to accept extra args, even if
	unsupported

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -321,6 +321,12 @@
     def descr_get_transpose(self, space):
         return W_NDimArray(self.implementation.transpose(self))
 
+    def descr_transpose(self, space, args_w):
+        if len(args_w) != 0:
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "axes unsupported for transpose"))
+        return self.descr_get_transpose(space)
+
     @unwrap_spec(axis1=int, axis2=int)
     def descr_swapaxes(self, space, axis1, axis2):
         """a.swapaxes(axis1, axis2)
@@ -859,7 +865,13 @@
     descr_cumprod = _reduce_ufunc_impl('multiply', cumultative=True)
 
     def _reduce_argmax_argmin_impl(op_name):
-        def impl(self, space):
+        def impl(self, space, w_axis=None, w_out=None):
+            if not space.is_none(w_axis):
+                raise OperationError(space.w_NotImplementedError, space.wrap(
+                    "axis unsupported for %s" % op_name))
+            if not space.is_none(w_out):
+                raise OperationError(space.w_NotImplementedError, space.wrap(
+                    "out unsupported for %s" % op_name))
             if self.get_size() == 0:
                 raise OperationError(space.w_ValueError,
                     space.wrap("Can't call %s on zero-size arrays" % op_name))
@@ -1130,7 +1142,7 @@
     copy = interp2app(W_NDimArray.descr_copy),
     reshape = interp2app(W_NDimArray.descr_reshape),
     T = GetSetProperty(W_NDimArray.descr_get_transpose),
-    transpose = interp2app(W_NDimArray.descr_get_transpose),
+    transpose = interp2app(W_NDimArray.descr_transpose),
     tolist = interp2app(W_NDimArray.descr_tolist),
     flatten = interp2app(W_NDimArray.descr_flatten),
     ravel = interp2app(W_NDimArray.descr_ravel),
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -680,8 +680,8 @@
         # strange
         assert dtype('string').str == '|S0'
         assert dtype('unicode').str == byteorder + 'U0'
-        assert dtype(('string', 7)).str == '|S7'
-        assert dtype(('unicode', 7)).str == '<U7'
+        #assert dtype(('string', 7)).str == '|S7'
+        #assert dtype(('unicode', 7)).str == '<U7'
 
     def test_intp(self):
         from numpypy import dtype
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1331,12 +1331,20 @@
         a = array([0, 0, 1])
         assert a.argmax() == 2
 
+        a = array([[1, 2], [3, 4], [5, 6]])
+        assert a.argmax() == 5
+        assert a.argmax(axis=None, out=None) == 5
+        assert a[:2, ].argmax() == 3
+        raises(NotImplementedError, a.argmax, axis=0)
+
     def test_argmin(self):
         from numpypy import array
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         assert a.argmin() == 3
+        assert a.argmin(axis=None, out=None) == 3
         b = array([])
         raises(ValueError, "b.argmin()")
+        raises(NotImplementedError, a.argmin, axis=0)
 
     def test_all(self):
         from numpypy import array
@@ -2146,12 +2154,6 @@
         c[:] = b
         assert (c == [[[12, 14], [12, 14]], [[12, 14], [12, 14]]]).all()
 
-    def test_argmax(self):
-        from numpypy import array
-        a = array([[1, 2], [3, 4], [5, 6]])
-        assert a.argmax() == 5
-        assert a[:2, ].argmax() == 3
-
     def test_broadcast_wrong_shapes(self):
         from numpypy import zeros
         a = zeros((4, 3, 2))
@@ -2188,6 +2190,7 @@
         b = a.T
         assert(b[:, 0] == a[0, :]).all()
         assert (a.transpose() == b).all()
+        raises(NotImplementedError, a.transpose, (1, 0, 2))
 
     def test_flatiter(self):
         from numpypy import array, flatiter, arange, zeros

From noreply at buildbot.pypy.org  Tue Oct 29 10:41:56 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 29 Oct 2013 10:41:56 +0100 (CET)
Subject: [pypy-commit] pypy rordereddict: finish porting
Message-ID: <20131029094156.7948A1C1050@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rordereddict
Changeset: r67676:bdb7bae76f74
Date: 2013-10-29 11:40 +0200
http://bitbucket.org/pypy/pypy/changeset/bdb7bae76f74/

Log:	finish porting

diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -300,6 +300,10 @@
     dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn)
     return SomeDict(dictdef)
 
+def robjmodel_r_ordereddict(s_eqfn, s_hashfn):
+    dictdef = getbookkeeper().getdictdef(is_r_dict=True)
+    dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn)
+    return SomeOrderedDict(dictdef)
 
 def robjmodel_hlinvoke(s_repr, s_llcallable, *args_s):
     from rpython.rtyper import rmodel
@@ -359,6 +363,7 @@
 BUILTIN_ANALYZERS[rpython.rlib.rarithmetic.longlongmask] = rarith_longlongmask
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.instantiate] = robjmodel_instantiate
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_dict] = robjmodel_r_dict
+BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_ordereddict] = robjmodel_r_ordereddict
 BUILTIN_ANALYZERS[OrderedDict] = lambda : SomeOrderedDict(getbookkeeper().getdictdef())
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.hlinvoke] = robjmodel_hlinvoke
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.keepalive_until_here] = robjmodel_keepalive_until_here
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -628,8 +628,11 @@
     The functions key_eq() and key_hash() are used by the key comparison
     algorithm."""
 
+    def _newdict(self):
+        return {}
+
     def __init__(self, key_eq, key_hash, force_non_null=False):
-        self._dict = {}
+        self._dict = self._newdict()
         self.key_eq = key_eq
         self.key_hash = key_hash
         self.force_non_null = force_non_null
@@ -664,7 +667,7 @@
         return dk.key, value
 
     def copy(self):
-        result = r_dict(self.key_eq, self.key_hash)
+        result = self.__class__(self.key_eq, self.key_hash)
         result.update(self)
         return result
 
@@ -700,6 +703,11 @@
     def __hash__(self):
         raise TypeError("cannot hash r_dict instances")
 
+class r_ordereddict(r_dict):
+    def _newdict(self):
+        from collections import OrderedDict
+        
+        return OrderedDict()
 
 class _r_dictkey(object):
     __slots__ = ['dic', 'key', 'hash']
@@ -735,7 +743,7 @@
     Function and staticmethod objects are duplicated, which means
     that annotation will not consider them as identical to another
     copy in another unrelated class.
-    
+
     By default, "special" methods and class attributes, with a name
     like "__xxx__", are not copied unless they are "__init__" or
     "__del__".  The list can be changed with the optional second
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -734,13 +734,23 @@
     hop.exception_cannot_occur()
     r_dict = hop.r_result
     cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    return hop.gendirectcall(ll_newdict, cDICT)
+    v_result = hop.gendirectcall(ll_newdict, cDICT)
+    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
+    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
+    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
+        hop.genop('setfield', [v_result, cname, v_eqfn])
+    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
+        hop.genop('setfield', [v_result, cname, v_hashfn])
+    return v_result
 
 BUILTIN_TYPER[objectmodel.instantiate] = rtype_instantiate
 BUILTIN_TYPER[isinstance] = rtype_builtin_isinstance
 BUILTIN_TYPER[hasattr] = rtype_builtin_hasattr
 BUILTIN_TYPER[objectmodel.r_dict] = rtype_r_dict
 BUILTIN_TYPER[OrderedDict] = rtype_ordered_dict
+BUILTIN_TYPER[objectmodel.r_ordereddict] = rtype_ordered_dict
 
 # _________________________________________________________________
 # weakrefs
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -594,42 +594,6 @@
         res = self.interpret(g, [3])
         assert res == 77
 
-    def test_r_dict(self):
-        class FooError(Exception):
-            pass
-        def myeq(n, m):
-            return n == m
-        def myhash(n):
-            if n < 0:
-                raise FooError
-            return -n
-        def f(n):
-            d = r_dict(myeq, myhash)
-            for i in range(10):
-                d[i] = i*i
-            try:
-                value1 = d[n]
-            except FooError:
-                value1 = 99
-            try:
-                value2 = n in d
-            except FooError:
-                value2 = 99
-            try:
-                value3 = d[-n]
-            except FooError:
-                value3 = 99
-            try:
-                value4 = (-n) in d
-            except FooError:
-                value4 = 99
-            return (value1 * 1000000 +
-                    value2 * 10000 +
-                    value3 * 100 +
-                    value4)
-        res = self.interpret(f, [5])
-        assert res == 25019999
-
     def test_resize_during_iteration(self):
         def func():
             d = self.newdict()
@@ -706,30 +670,6 @@
         res = self.interpret(func, [])
         assert res in [5263, 6352]
 
-    def test_dict_popitem_hash(self):
-        def deq(n, m):
-            return n == m
-        def dhash(n):
-            return ~n
-        def func():
-            d = r_dict(deq, dhash)
-            d[5] = 2
-            d[6] = 3
-            k1, v1 = d.popitem()
-            assert len(d) == 1
-            k2, v2 = d.popitem()
-            try:
-                d.popitem()
-            except KeyError:
-                pass
-            else:
-                assert 0, "should have raised KeyError"
-            assert len(d) == 0
-            return k1*1000 + v1*100 + k2*10 + v2
-
-        res = self.interpret(func, [])
-        assert res in [5263, 6352]
-
     def test_dict_pop(self):
         def f(n, default):
             d = self.newdict()
@@ -777,74 +717,10 @@
         res = self.interpret(func, [6])
         assert res == 0
 
-    def test_deleted_entry_reusage_with_colliding_hashes(self):
-        def lowlevelhash(value):
-            p = rstr.mallocstr(len(value))
-            for i in range(len(value)):
-                p.chars[i] = value[i]
-            return rstr.LLHelpers.ll_strhash(p)
-
-        def func(c1, c2):
-            c1 = chr(c1)
-            c2 = chr(c2)
-            d = self.newdict()
-            d[c1] = 1
-            d[c2] = 2
-            del d[c1]
-            return d[c2]
-
-        char_by_hash = {}
-        base = rdict.DICT_INITSIZE
-        for y in range(0, 256):
-            y = chr(y)
-            y_hash = lowlevelhash(y) % base
-            char_by_hash.setdefault(y_hash, []).append(y)
-
-        x, y = char_by_hash[0][:2]   # find a collision
-
-        res = self.interpret(func, [ord(x), ord(y)])
-        assert res == 2
-
-        def func2(c1, c2):
-            c1 = chr(c1)
-            c2 = chr(c2)
-            d = {}
-            d[c1] = 1
-            d[c2] = 2
-            del d[c1]
-            d[c1] = 3
-            return d
-
-        res = self.interpret(func2, [ord(x), ord(y)])
-        for i in range(len(res.entries)):
-            assert not (res.entries.everused(i) and not res.entries.valid(i))
-
-        def func3(c0, c1, c2, c3, c4, c5, c6, c7):
-            d = {}
-            c0 = chr(c0) ; d[c0] = 1; del d[c0]
-            c1 = chr(c1) ; d[c1] = 1; del d[c1]
-            c2 = chr(c2) ; d[c2] = 1; del d[c2]
-            c3 = chr(c3) ; d[c3] = 1; del d[c3]
-            c4 = chr(c4) ; d[c4] = 1; del d[c4]
-            c5 = chr(c5) ; d[c5] = 1; del d[c5]
-            c6 = chr(c6) ; d[c6] = 1; del d[c6]
-            c7 = chr(c7) ; d[c7] = 1; del d[c7]
-            return d
-
-        if rdict.DICT_INITSIZE != 8:
-            py.test.skip("make dict tests more indepdent from initsize")
-        res = self.interpret(func3, [ord(char_by_hash[i][0])
-                                   for i in range(rdict.DICT_INITSIZE)])
-        count_frees = 0
-        for i in range(len(res.entries)):
-            if not res.entries.everused(i):
-                count_frees += 1
-        assert count_frees >= 3
-
     def test_dict_valid_resize(self):
         # see if we find our keys after resize
         def func():
-            d = {}
+            d = self.newdict()
             # fill it up
             for i in range(10):
                 d[str(i)] = 0
@@ -939,25 +815,6 @@
         assert self.interpret(func, [5]) == 123
         assert self.interpret(func, [42]) == 321
 
-    def test_nonnull_hint(self):
-        def eq(a, b):
-            return a == b
-        def rhash(a):
-            return 3
-
-        def func(i):
-            d = r_dict(eq, rhash, force_non_null=True)
-            if not i:
-                d[None] = i
-            else:
-                d[str(i)] = i
-            return "12" in d, d
-
-        llres = self.interpret(func, [12])
-        assert llres.item0 == 1
-        DICT = lltype.typeOf(llres.item1)
-        assert sorted(DICT.TO.entries.TO.OF._flds) == ['f_hash', 'key', 'value']
-
     def test_memoryerror_should_not_insert(self):
         # This shows a misbehaviour that also exists in CPython 2.7, but not
         # any more in CPython 3.3.  The behaviour is that even if a dict
@@ -1160,6 +1017,149 @@
         assert lltype.typeOf(res.item1) == lltype.typeOf(res.item2)
         assert lltype.typeOf(res.item1) == lltype.typeOf(res.item3)
 
+    def test_r_dict(self):
+        class FooError(Exception):
+            pass
+        def myeq(n, m):
+            return n == m
+        def myhash(n):
+            if n < 0:
+                raise FooError
+            return -n
+        def f(n):
+            d = r_dict(myeq, myhash)
+            for i in range(10):
+                d[i] = i*i
+            try:
+                value1 = d[n]
+            except FooError:
+                value1 = 99
+            try:
+                value2 = n in d
+            except FooError:
+                value2 = 99
+            try:
+                value3 = d[-n]
+            except FooError:
+                value3 = 99
+            try:
+                value4 = (-n) in d
+            except FooError:
+                value4 = 99
+            return (value1 * 1000000 +
+                    value2 * 10000 +
+                    value3 * 100 +
+                    value4)
+        res = self.interpret(f, [5])
+        assert res == 25019999
+
+    def test_dict_popitem_hash(self):
+        def deq(n, m):
+            return n == m
+        def dhash(n):
+            return ~n
+        def func():
+            d = r_dict(deq, dhash)
+            d[5] = 2
+            d[6] = 3
+            k1, v1 = d.popitem()
+            assert len(d) == 1
+            k2, v2 = d.popitem()
+            try:
+                d.popitem()
+            except KeyError:
+                pass
+            else:
+                assert 0, "should have raised KeyError"
+            assert len(d) == 0
+            return k1*1000 + v1*100 + k2*10 + v2
+
+        res = self.interpret(func, [])
+        assert res in [5263, 6352]
+
+    def test_nonnull_hint(self):
+        def eq(a, b):
+            return a == b
+        def rhash(a):
+            return 3
+
+        def func(i):
+            d = r_dict(eq, rhash, force_non_null=True)
+            if not i:
+                d[None] = i
+            else:
+                d[str(i)] = i
+            return "12" in d, d
+
+        llres = self.interpret(func, [12])
+        assert llres.item0 == 1
+        DICT = lltype.typeOf(llres.item1)
+        assert sorted(DICT.TO.entries.TO.OF._flds) == ['f_hash', 'key', 'value']
+
+    def test_deleted_entry_reusage_with_colliding_hashes(self):
+        def lowlevelhash(value):
+            p = rstr.mallocstr(len(value))
+            for i in range(len(value)):
+                p.chars[i] = value[i]
+            return rstr.LLHelpers.ll_strhash(p)
+
+        def func(c1, c2):
+            c1 = chr(c1)
+            c2 = chr(c2)
+            d = self.newdict()
+            d[c1] = 1
+            d[c2] = 2
+            del d[c1]
+            return d[c2]
+
+        char_by_hash = {}
+        base = rdict.DICT_INITSIZE
+        for y in range(0, 256):
+            y = chr(y)
+            y_hash = lowlevelhash(y) % base
+            char_by_hash.setdefault(y_hash, []).append(y)
+
+        x, y = char_by_hash[0][:2]   # find a collision
+
+        res = self.interpret(func, [ord(x), ord(y)])
+        assert res == 2
+
+        def func2(c1, c2):
+            c1 = chr(c1)
+            c2 = chr(c2)
+            d = self.newdict()
+            d[c1] = 1
+            d[c2] = 2
+            del d[c1]
+            d[c1] = 3
+            return d
+
+        res = self.interpret(func2, [ord(x), ord(y)])
+        for i in range(len(res.entries)):
+            assert not (res.entries.everused(i) and not res.entries.valid(i))
+
+        def func3(c0, c1, c2, c3, c4, c5, c6, c7):
+            d = self.newdict()
+            c0 = chr(c0) ; d[c0] = 1; del d[c0]
+            c1 = chr(c1) ; d[c1] = 1; del d[c1]
+            c2 = chr(c2) ; d[c2] = 1; del d[c2]
+            c3 = chr(c3) ; d[c3] = 1; del d[c3]
+            c4 = chr(c4) ; d[c4] = 1; del d[c4]
+            c5 = chr(c5) ; d[c5] = 1; del d[c5]
+            c6 = chr(c6) ; d[c6] = 1; del d[c6]
+            c7 = chr(c7) ; d[c7] = 1; del d[c7]
+            return d
+
+        if rdict.DICT_INITSIZE != 8:
+            py.test.skip("make dict tests more indepdent from initsize")
+        res = self.interpret(func3, [ord(char_by_hash[i][0])
+                                   for i in range(rdict.DICT_INITSIZE)])
+        count_frees = 0
+        for i in range(len(res.entries)):
+            if not res.entries.everused(i):
+                count_frees += 1
+        assert count_frees >= 3
+
 class TestStress:
 
     def test_stress(self):
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -6,6 +6,7 @@
 from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.annlowlevel import llstr, hlstr
 from rpython.rtyper.test.test_rdict import BaseTestRDict
+from rpython.rlib import objectmodel
 
 
 def get_indexes(ll_d):
@@ -275,3 +276,64 @@
 
     def test_memoryerror_should_not_insert(self):
         py.test.skip("I don't want to edit this file on two branches")
+
+
+    def test_r_dict(self):
+        class FooError(Exception):
+            pass
+        def myeq(n, m):
+            return n == m
+        def myhash(n):
+            if n < 0:
+                raise FooError
+            return -n
+        def f(n):
+            d = objectmodel.r_ordereddict(myeq, myhash)
+            for i in range(10):
+                d[i] = i*i
+            try:
+                value1 = d[n]
+            except FooError:
+                value1 = 99
+            try:
+                value2 = n in d
+            except FooError:
+                value2 = 99
+            try:
+                value3 = d[-n]
+            except FooError:
+                value3 = 99
+            try:
+                value4 = (-n) in d
+            except FooError:
+                value4 = 99
+            return (value1 * 1000000 +
+                    value2 * 10000 +
+                    value3 * 100 +
+                    value4)
+        res = self.interpret(f, [5])
+        assert res == 25019999
+
+    def test_dict_popitem_hash(self):
+        def deq(n, m):
+            return n == m
+        def dhash(n):
+            return ~n
+        def func():
+            d = objectmodel.r_ordereddict(deq, dhash)
+            d[5] = 2
+            d[6] = 3
+            k1, v1 = d.popitem()
+            assert len(d) == 1
+            k2, v2 = d.popitem()
+            try:
+                d.popitem()
+            except KeyError:
+                pass
+            else:
+                assert 0, "should have raised KeyError"
+            assert len(d) == 0
+            return k1*1000 + v1*100 + k2*10 + v2
+
+        res = self.interpret(func, [])
+        assert res in [5263, 6352]

From noreply at buildbot.pypy.org  Tue Oct 29 10:45:34 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 29 Oct 2013 10:45:34 +0100 (CET)
Subject: [pypy-commit] pypy rordereddict: close about to be merged branch
Message-ID: <20131029094534.443F41C1050@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rordereddict
Changeset: r67677:728470d92f02
Date: 2013-10-29 11:42 +0200
http://bitbucket.org/pypy/pypy/changeset/728470d92f02/

Log:	close about to be merged branch


From noreply at buildbot.pypy.org  Tue Oct 29 10:45:35 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 29 Oct 2013 10:45:35 +0100 (CET)
Subject: [pypy-commit] pypy default: merge rordereddict - it imports
 rdict.py from rdict-experiments-3 and uses
Message-ID: <20131029094535.EE8B41C1050@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67678:3d5211efc4df
Date: 2013-10-29 11:44 +0200
http://bitbucket.org/pypy/pypy/changeset/3d5211efc4df/

Log:	merge rordereddict - it imports rdict.py from rdict-experiments-3
	and uses it to provide OrderedDict in RPython

diff too long, truncating to 2000 out of 2944 lines

diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -6,7 +6,8 @@
 import operator
 from rpython.tool.pairtype import pair, pairtype
 from rpython.annotator.model import SomeObject, SomeInteger, SomeBool, s_Bool
-from rpython.annotator.model import SomeString, SomeChar, SomeList, SomeDict
+from rpython.annotator.model import SomeString, SomeChar, SomeList, SomeDict,\
+     SomeOrderedDict
 from rpython.annotator.model import SomeUnicodeCodePoint, SomeUnicodeString
 from rpython.annotator.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue
 from rpython.annotator.model import SomeInstance, SomeBuiltin, SomeIterator
@@ -581,7 +582,8 @@
 class __extend__(pairtype(SomeDict, SomeDict)):
 
     def union((dic1, dic2)):
-        return SomeDict(dic1.dictdef.union(dic2.dictdef))
+        assert dic1.__class__ == dic2.__class__
+        return dic1.__class__(dic1.dictdef.union(dic2.dictdef))
 
 
 class __extend__(pairtype(SomeDict, SomeObject)):
@@ -840,6 +842,7 @@
 _make_none_union('SomeString',      'no_nul=obj.no_nul, can_be_None=True')
 _make_none_union('SomeUnicodeString', 'can_be_None=True')
 _make_none_union('SomeList',         'obj.listdef')
+_make_none_union('SomeOrderedDict',          'obj.dictdef')
 _make_none_union('SomeDict',          'obj.dictdef')
 _make_none_union('SomeWeakRef',         'obj.classdef')
 
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -5,9 +5,10 @@
 from __future__ import absolute_import
 
 import sys, types, inspect, weakref
+from collections import OrderedDict
 
 from rpython.flowspace.model import Constant
-from rpython.annotator.model import (
+from rpython.annotator.model import (SomeOrderedDict,
     SomeString, SomeChar, SomeFloat, SomePtr, unionof, SomeInstance, SomeDict,
     SomeBuiltin, SomePBC, SomeInteger, TLS, SomeAddress, SomeUnicodeCodePoint,
     s_None, s_ImpossibleValue, SomeLLADTMeth, SomeBool, SomeTuple,
@@ -370,7 +371,7 @@
                 for e in x:
                     listdef.generalize(self.immutablevalue(e, False))
                 result = SomeList(listdef)
-        elif tp is dict or tp is r_dict:
+        elif tp is dict or tp is r_dict or tp is OrderedDict:
             if need_const:
                 key = Constant(x)
                 try:
@@ -412,7 +413,10 @@
                     dictdef.generalize_key(self.immutablevalue(ek, False))
                     dictdef.generalize_value(self.immutablevalue(ev, False))
                     dictdef.seen_prebuilt_key(ek)
-                result = SomeDict(dictdef)
+                if tp is OrderedDict:
+                    result = SomeOrderedDict(dictdef)
+                else:
+                    result = SomeDict(dictdef)
         elif tp is weakref.ReferenceType:
             x1 = x()
             if x1 is None:
diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -2,11 +2,13 @@
 Built-in functions.
 """
 import sys
+from collections import OrderedDict
 
 from rpython.annotator.model import (
     SomeInteger, SomeObject, SomeChar, SomeBool, SomeString, SomeTuple, s_Bool,
     SomeUnicodeCodePoint, SomeAddress, SomeFloat, unionof, SomeUnicodeString,
     SomePBC, SomeInstance, SomeDict, SomeList, SomeWeakRef, SomeIterator,
+    SomeOrderedDict,
     SomeByteArray, annotation_to_lltype, lltype_to_annotation,
     ll_to_annotation, add_knowntypedata, s_ImpossibleValue,)
 from rpython.annotator.bookkeeper import getbookkeeper
@@ -298,6 +300,10 @@
     dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn)
     return SomeDict(dictdef)
 
+def robjmodel_r_ordereddict(s_eqfn, s_hashfn):
+    dictdef = getbookkeeper().getdictdef(is_r_dict=True)
+    dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn)
+    return SomeOrderedDict(dictdef)
 
 def robjmodel_hlinvoke(s_repr, s_llcallable, *args_s):
     from rpython.rtyper import rmodel
@@ -357,6 +363,8 @@
 BUILTIN_ANALYZERS[rpython.rlib.rarithmetic.longlongmask] = rarith_longlongmask
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.instantiate] = robjmodel_instantiate
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_dict] = robjmodel_r_dict
+BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_ordereddict] = robjmodel_r_ordereddict
+BUILTIN_ANALYZERS[OrderedDict] = lambda : SomeOrderedDict(getbookkeeper().getdictdef())
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.hlinvoke] = robjmodel_hlinvoke
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.keepalive_until_here] = robjmodel_keepalive_until_here
 BUILTIN_ANALYZERS[rpython.rtyper.lltypesystem.llmemory.cast_ptr_to_adr] = llmemory_cast_ptr_to_adr
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -32,6 +32,7 @@
 import inspect
 import weakref
 from types import BuiltinFunctionType, MethodType
+from collections import OrderedDict
 
 import rpython
 from rpython.tool import descriptor
@@ -355,6 +356,18 @@
         else:
             return '{...%s...}' % (len(const),)
 
+class SomeOrderedDict(SomeDict):
+    knowntype = OrderedDict
+
+    def method_copy(dct):
+        return SomeOrderedDict(dct.dictdef)
+
+    def method_update(dct1, dct2):
+        if s_None.contains(dct2):
+            return SomeImpossibleValue()
+        assert isinstance(dct2, SomeOrderedDict), "OrderedDict.update(dict) not allowed"
+        dct1.dictdef.union(dct2.dictdef)
+
 
 class SomeIterator(SomeObject):
     "Stands for an iterator returning objects from a given container."
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -628,8 +628,11 @@
     The functions key_eq() and key_hash() are used by the key comparison
     algorithm."""
 
+    def _newdict(self):
+        return {}
+
     def __init__(self, key_eq, key_hash, force_non_null=False):
-        self._dict = {}
+        self._dict = self._newdict()
         self.key_eq = key_eq
         self.key_hash = key_hash
         self.force_non_null = force_non_null
@@ -664,7 +667,7 @@
         return dk.key, value
 
     def copy(self):
-        result = r_dict(self.key_eq, self.key_hash)
+        result = self.__class__(self.key_eq, self.key_hash)
         result.update(self)
         return result
 
@@ -700,6 +703,11 @@
     def __hash__(self):
         raise TypeError("cannot hash r_dict instances")
 
+class r_ordereddict(r_dict):
+    def _newdict(self):
+        from collections import OrderedDict
+        
+        return OrderedDict()
 
 class _r_dictkey(object):
     __slots__ = ['dic', 'key', 'hash']
@@ -735,7 +743,7 @@
     Function and staticmethod objects are duplicated, which means
     that annotation will not consider them as identical to another
     copy in another unrelated class.
-    
+
     By default, "special" methods and class attributes, with a name
     like "__xxx__", are not copied unless they are "__init__" or
     "__del__".  The list can be changed with the optional second
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -522,8 +522,10 @@
     A = lltype.typeOf(source)
     assert A == lltype.typeOf(dest)
     if isinstance(A.TO, lltype.GcArray):
-        assert isinstance(A.TO.OF, lltype.Ptr)
-        assert A.TO.OF.TO._gckind == 'gc'
+        if isinstance(A.TO.OF, lltype.Ptr):
+            assert A.TO.OF.TO._gckind == 'gc'
+        else:
+            assert isinstance(A.TO.OF, lltype.Struct)
     else:
         assert isinstance(A.TO, lltype.GcStruct)
         assert A.TO._arrayfld is not None
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -0,0 +1,1149 @@
+import sys
+from rpython.tool.pairtype import pairtype
+from rpython.flowspace.model import Constant
+from rpython.rtyper.rdict import AbstractDictRepr, AbstractDictIteratorRepr
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rlib import objectmodel, jit, rgc
+from rpython.rlib.debug import ll_assert
+from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rtyper import rmodel
+from rpython.rtyper.error import TyperError
+from rpython.rtyper.annlowlevel import llhelper
+
+
+# ____________________________________________________________
+#
+#  generic implementation of RPython dictionary, with parametric DICTKEY and
+#  DICTVALUE types. The basic implementation is a sparse array of indexes
+#  plus a dense array of structs that contain keys and values. struct looks
+#  like that:
+#
+#
+#    struct dictentry {
+#        DICTKEY key;
+#        DICTVALUE value;
+#        long f_hash;        # (optional) key hash, if hard to recompute
+#        bool f_valid;      # (optional) the entry is filled
+#    }
+#
+#    struct dicttable {
+#        int num_items;
+#        int num_used_items;
+#        int resize_counter;
+#        {byte, short, int, long} *indexes;
+#        dictentry *entries;
+#        lookup_function_no; # one of the four possible functions for different
+#                         # size dicts
+#        (Function DICTKEY, DICTKEY -> bool) *fnkeyeq;
+#        (Function DICTKEY -> int) *fnkeyhash;
+#    }
+#
+#
+
+def ll_call_lookup_function(d, key, hash, flag):
+    DICT = lltype.typeOf(d).TO
+    fun = d.lookup_function_no
+    if fun == FUNC_BYTE:
+        return DICT.lookup_family.byte_lookup_function(d, key, hash, flag)
+    elif fun == FUNC_SHORT:
+        return DICT.lookup_family.short_lookup_function(d, key, hash, flag)
+    elif IS_64BIT and fun == FUNC_INT:
+        return DICT.lookup_family.int_lookup_function(d, key, hash, flag)
+    elif fun == FUNC_LONG:
+        return DICT.lookup_family.long_lookup_function(d, key, hash, flag)
+    assert False
+
+def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
+                ll_fasthash_function=None, ll_hash_function=None,
+                ll_eq_function=None, method_cache={},
+                dummykeyobj=None, dummyvalueobj=None, rtyper=None,
+                setup_lookup_funcs=True):
+    # get the actual DICT type. if DICT is None, it's created, otherwise
+    # forward reference is becoming DICT
+    if DICT is None:
+        DICT = lltype.GcForwardReference()
+    # compute the shape of the DICTENTRY structure
+    entryfields = []
+    entrymeths = {
+        'allocate': lltype.typeMethod(_ll_malloc_entries),
+        'delete': _ll_free_entries,
+        'must_clear_key':   (isinstance(DICTKEY, lltype.Ptr)
+                             and DICTKEY._needsgc()),
+        'must_clear_value': (isinstance(DICTVALUE, lltype.Ptr)
+                             and DICTVALUE._needsgc()),
+        }
+
+    # * the key
+    entryfields.append(("key", DICTKEY))
+
+    # * the state of the entry - trying to encode it as dummy objects
+    if dummykeyobj:
+        # all the state can be encoded in the key
+        entrymeths['dummy_obj'] = dummykeyobj
+        entrymeths['valid'] = ll_valid_from_key
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_key
+        # the key is overwritten by 'dummy' when the entry is deleted
+        entrymeths['must_clear_key'] = False
+
+    elif dummyvalueobj:
+        # all the state can be encoded in the value
+        entrymeths['dummy_obj'] = dummyvalueobj
+        entrymeths['valid'] = ll_valid_from_value
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_value
+        # value is overwritten by 'dummy' when entry is deleted
+        entrymeths['must_clear_value'] = False
+
+    else:
+        # we need a flag to know if the entry was ever used
+        entryfields.append(("f_valid", lltype.Bool))
+        entrymeths['valid'] = ll_valid_from_flag
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_flag
+
+    # * the value
+    entryfields.append(("value", DICTVALUE))
+
+    if ll_fasthash_function is None:
+        entryfields.append(("f_hash", lltype.Signed))
+        entrymeths['hash'] = ll_hash_from_cache
+    else:
+        entrymeths['hash'] = ll_hash_recomputed
+        entrymeths['fasthashfn'] = ll_fasthash_function
+
+    # Build the lltype data structures
+    DICTENTRY = lltype.Struct("dictentry", *entryfields)
+    DICTENTRYARRAY = lltype.GcArray(DICTENTRY,
+                                    adtmeths=entrymeths)
+    fields =          [ ("num_items", lltype.Signed),
+                        ("num_used_items", lltype.Signed),
+                        ("resize_counter", lltype.Signed),
+                        ("indexes", llmemory.GCREF),
+                        ("lookup_function_no", lltype.Signed),
+                        ("entries", lltype.Ptr(DICTENTRYARRAY)) ]
+    if get_custom_eq_hash is not None:
+        r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash()
+        fields.extend([ ("fnkeyeq", r_rdict_eqfn.lowleveltype),
+                        ("fnkeyhash", r_rdict_hashfn.lowleveltype) ])
+        adtmeths = {
+            'keyhash':        ll_keyhash_custom,
+            'keyeq':          ll_keyeq_custom,
+            'r_rdict_eqfn':   r_rdict_eqfn,
+            'r_rdict_hashfn': r_rdict_hashfn,
+            'paranoia':       True,
+            }
+    else:
+        # figure out which functions must be used to hash and compare
+        ll_keyhash = ll_hash_function
+        ll_keyeq = ll_eq_function
+        ll_keyhash = lltype.staticAdtMethod(ll_keyhash)
+        if ll_keyeq is not None:
+            ll_keyeq = lltype.staticAdtMethod(ll_keyeq)
+        adtmeths = {
+            'keyhash':  ll_keyhash,
+            'keyeq':    ll_keyeq,
+            'paranoia': False,
+            }
+    adtmeths['KEY']   = DICTKEY
+    adtmeths['VALUE'] = DICTVALUE
+    adtmeths['lookup_function'] = lltype.staticAdtMethod(ll_call_lookup_function)
+    adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
+
+    family = LookupFamily()
+    adtmeths['lookup_family'] = family
+
+    DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
+                                *fields))
+
+    family.empty_array = DICTENTRYARRAY.allocate(0)
+    if setup_lookup_funcs:
+        _setup_lookup_funcs(DICT, rtyper, family)
+    return DICT
+
+def _setup_lookup_funcs(DICT, rtyper, family):
+    DICTKEY = DICT.entries.TO.OF.key
+    LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY,
+                                              lltype.Signed, lltype.Signed],
+                                              lltype.Signed))
+
+
+    STORECLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
+                                                  lltype.Signed,
+                                                  lltype.Signed],
+                                                 lltype.Void))
+
+    for name, T in [('byte', rffi.UCHAR),
+                    ('short', rffi.USHORT),
+                    ('int', rffi.UINT),
+                    ('long', lltype.Unsigned)]:
+        if name == 'int' and not IS_64BIT:
+            continue
+        lookupfn, storecleanfn = new_lookup_functions(LOOKUP_FUNC,
+                                                      STORECLEAN_FUNC, T=T,
+                                                      rtyper=rtyper)
+        setattr(family, '%s_lookup_function' % name, lookupfn)
+        setattr(family, '%s_insert_clean_function' % name, storecleanfn)
+
+def llhelper_or_compile(rtyper, FUNCPTR, ll_func):
+    # the check is for pseudo rtyper from tests
+    if rtyper is None or not hasattr(rtyper, 'annotate_helper_fn'):
+        return llhelper(FUNCPTR, ll_func)
+    else:
+        return rtyper.annotate_helper_fn(ll_func, FUNCPTR.TO.ARGS)
+
+class LookupFamily:
+    def _freeze_(self):
+        return True
+
+
+class OrderedDictRepr(AbstractDictRepr):
+
+    def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
+                 custom_eq_hash=None, force_non_null=False):
+        assert not force_non_null
+        self.rtyper = rtyper
+        self.finalized = False
+        self.DICT = lltype.GcForwardReference()
+        self.lowleveltype = lltype.Ptr(self.DICT)
+        self.custom_eq_hash = custom_eq_hash is not None
+        if not isinstance(key_repr, rmodel.Repr):  # not computed yet, done by setup()
+            assert callable(key_repr)
+            self._key_repr_computer = key_repr
+        else:
+            self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr)
+        if not isinstance(value_repr, rmodel.Repr):  # not computed yet, done by setup()
+            assert callable(value_repr)
+            self._value_repr_computer = value_repr
+        else:
+            self.external_value_repr, self.value_repr = self.pickrepr(value_repr)
+        self.dictkey = dictkey
+        self.dictvalue = dictvalue
+        self.dict_cache = {}
+        self._custom_eq_hash_repr = custom_eq_hash
+        # setup() needs to be called to finish this initialization
+
+    def _externalvsinternal(self, rtyper, item_repr):
+        return rmodel.externalvsinternal(self.rtyper, item_repr)
+
+    def _setup_repr(self):
+        if 'key_repr' not in self.__dict__:
+            key_repr = self._key_repr_computer()
+            self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr)
+        if 'value_repr' not in self.__dict__:
+            self.external_value_repr, self.value_repr = self.pickrepr(self._value_repr_computer())
+        if isinstance(self.DICT, lltype.GcForwardReference):
+            DICTKEY = self.key_repr.lowleveltype
+            DICTVALUE = self.value_repr.lowleveltype
+            # * we need an explicit flag if the key and the value is not
+            #   able to store dummy values
+            s_key   = self.dictkey.s_value
+            s_value = self.dictvalue.s_value
+            kwd = {}
+            if self.custom_eq_hash:
+                self.r_rdict_eqfn, self.r_rdict_hashfn = (
+                    self._custom_eq_hash_repr())
+                kwd['get_custom_eq_hash'] = self._custom_eq_hash_repr
+            else:
+                kwd['ll_hash_function'] = self.key_repr.get_ll_hash_function()
+                kwd['ll_eq_function'] = self.key_repr.get_ll_eq_function()
+                kwd['ll_fasthash_function'] = self.key_repr.get_ll_fasthash_function()
+            kwd['dummykeyobj'] = self.key_repr.get_ll_dummyval_obj(self.rtyper,
+                                                                   s_key)
+            kwd['dummyvalueobj'] = self.value_repr.get_ll_dummyval_obj(
+                self.rtyper, s_value)
+
+            kwd['setup_lookup_funcs'] = False
+            get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT,
+                        rtyper=self.rtyper, **kwd)
+
+    def _setup_repr_final(self):
+        if not self.finalized:
+            family = self.lowleveltype.TO.lookup_family
+            _setup_lookup_funcs(self.lowleveltype.TO, self.rtyper, family)
+            self.finalized = True
+
+
+    def convert_const(self, dictobj):
+        from rpython.rtyper.lltypesystem import llmemory
+        # get object from bound dict methods
+        #dictobj = getattr(dictobj, '__self__', dictobj)
+        if dictobj is None:
+            return lltype.nullptr(self.DICT)
+        if not isinstance(dictobj, (dict, objectmodel.r_dict)):
+            raise TypeError("expected a dict: %r" % (dictobj,))
+        try:
+            key = Constant(dictobj)
+            return self.dict_cache[key]
+        except KeyError:
+            self.setup()
+            self.setup_final()
+            l_dict = ll_newdict_size(self.DICT, len(dictobj))
+            self.dict_cache[key] = l_dict
+            r_key = self.key_repr
+            if r_key.lowleveltype == llmemory.Address:
+                raise TypeError("No prebuilt dicts of address keys")
+            r_value = self.value_repr
+            if isinstance(dictobj, objectmodel.r_dict):
+                if self.r_rdict_eqfn.lowleveltype != lltype.Void:
+                    l_fn = self.r_rdict_eqfn.convert_const(dictobj.key_eq)
+                    l_dict.fnkeyeq = l_fn
+                if self.r_rdict_hashfn.lowleveltype != lltype.Void:
+                    l_fn = self.r_rdict_hashfn.convert_const(dictobj.key_hash)
+                    l_dict.fnkeyhash = l_fn
+
+                for dictkeycontainer, dictvalue in dictobj._dict.items():
+                    llkey = r_key.convert_const(dictkeycontainer.key)
+                    llvalue = r_value.convert_const(dictvalue)
+                    _ll_dict_insertclean(l_dict, llkey, llvalue,
+                                         dictkeycontainer.hash)
+                return l_dict
+
+            else:
+                for dictkey, dictvalue in dictobj.items():
+                    llkey = r_key.convert_const(dictkey)
+                    llvalue = r_value.convert_const(dictvalue)
+                    _ll_dict_insertclean(l_dict, llkey, llvalue,
+                                         l_dict.keyhash(llkey))
+                return l_dict
+
+    def rtype_len(self, hop):
+        v_dict, = hop.inputargs(self)
+        return hop.gendirectcall(ll_dict_len, v_dict)
+
+    def rtype_bool(self, hop):
+        v_dict, = hop.inputargs(self)
+        return hop.gendirectcall(ll_dict_bool, v_dict)
+
+    def make_iterator_repr(self, *variant):
+        return DictIteratorRepr(self, *variant)
+
+    def rtype_method_get(self, hop):
+        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                 self.value_repr)
+        hop.exception_cannot_occur()
+        v_res = hop.gendirectcall(ll_dict_get, v_dict, v_key, v_default)
+        return self.recast_value(hop.llops, v_res)
+
+    def rtype_method_setdefault(self, hop):
+        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                 self.value_repr)
+        hop.exception_cannot_occur()
+        v_res = hop.gendirectcall(ll_dict_setdefault, v_dict, v_key, v_default)
+        return self.recast_value(hop.llops, v_res)
+
+    def rtype_method_copy(self, hop):
+        v_dict, = hop.inputargs(self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_copy, v_dict)
+
+    def rtype_method_update(self, hop):
+        v_dic1, v_dic2 = hop.inputargs(self, self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_update, v_dic1, v_dic2)
+
+    def _rtype_method_kvi(self, hop, ll_func):
+        v_dic, = hop.inputargs(self)
+        r_list = hop.r_result
+        cLIST = hop.inputconst(lltype.Void, r_list.lowleveltype.TO)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_func, cLIST, v_dic)
+
+    def rtype_method_keys(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_keys)
+
+    def rtype_method_values(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_values)
+
+    def rtype_method_items(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_items)
+
+    def rtype_method_iterkeys(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "keys").newiter(hop)
+
+    def rtype_method_itervalues(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "values").newiter(hop)
+
+    def rtype_method_iteritems(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "items").newiter(hop)
+
+    def rtype_method_clear(self, hop):
+        v_dict, = hop.inputargs(self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_clear, v_dict)
+
+    def rtype_method_popitem(self, hop):
+        v_dict, = hop.inputargs(self)
+        r_tuple = hop.r_result
+        cTUPLE = hop.inputconst(lltype.Void, r_tuple.lowleveltype)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_popitem, cTUPLE, v_dict)
+
+    def rtype_method_pop(self, hop):
+        if hop.nb_args == 2:
+            v_args = hop.inputargs(self, self.key_repr)
+            target = ll_dict_pop
+        elif hop.nb_args == 3:
+            v_args = hop.inputargs(self, self.key_repr, self.value_repr)
+            target = ll_dict_pop_default
+        hop.exception_is_here()
+        v_res = hop.gendirectcall(target, *v_args)
+        return self.recast_value(hop.llops, v_res)
+
+class __extend__(pairtype(OrderedDictRepr, rmodel.Repr)):
+
+    def rtype_getitem((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        if not r_dict.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)   # record that we know about it
+        hop.exception_is_here()
+        v_res = hop.gendirectcall(ll_dict_getitem, v_dict, v_key)
+        return r_dict.recast_value(hop.llops, v_res)
+
+    def rtype_delitem((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        if not r_dict.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)   # record that we know about it
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_delitem, v_dict, v_key)
+
+    def rtype_setitem((r_dict, r_key), hop):
+        v_dict, v_key, v_value = hop.inputargs(r_dict, r_dict.key_repr, r_dict.value_repr)
+        if r_dict.custom_eq_hash:
+            hop.exception_is_here()
+        else:
+            hop.exception_cannot_occur()
+        hop.gendirectcall(ll_dict_setitem, v_dict, v_key, v_value)
+
+    def rtype_contains((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_contains, v_dict, v_key)
+
+class __extend__(pairtype(OrderedDictRepr, OrderedDictRepr)):
+    def convert_from_to((r_dict1, r_dict2), v, llops):
+        # check that we don't convert from Dicts with
+        # different key/value types
+        if r_dict1.dictkey is None or r_dict2.dictkey is None:
+            return NotImplemented
+        if r_dict1.dictkey is not r_dict2.dictkey:
+            return NotImplemented
+        if r_dict1.dictvalue is None or r_dict2.dictvalue is None:
+            return NotImplemented
+        if r_dict1.dictvalue is not r_dict2.dictvalue:
+            return NotImplemented
+        return v
+
+# ____________________________________________________________
+#
+#  Low-level methods.  These can be run for testing, but are meant to
+#  be direct_call'ed from rtyped flow graphs, which means that they will
+#  get flowed and annotated, mostly with SomePtr.
+
+DICTINDEX_LONG = lltype.Ptr(lltype.GcArray(lltype.Unsigned))
+DICTINDEX_INT = lltype.Ptr(lltype.GcArray(rffi.UINT))
+DICTINDEX_SHORT = lltype.Ptr(lltype.GcArray(rffi.USHORT))
+DICTINDEX_BYTE = lltype.Ptr(lltype.GcArray(rffi.UCHAR))
+
+IS_64BIT = sys.maxint != 2 ** 31 - 1
+
+if IS_64BIT:
+    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG = range(4)
+else:
+    FUNC_BYTE, FUNC_SHORT, FUNC_LONG = range(3)
+
+def ll_malloc_indexes_and_choose_lookup(d, n):
+    if n <= 256:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_BYTE.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_BYTE
+    elif n <= 65536:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_SHORT.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_SHORT
+    elif IS_64BIT and n <= 2 ** 32:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_INT.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_INT
+    else:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_LONG.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_LONG
+
+def ll_call_insert_clean_function(d, hash, i):
+    DICT = lltype.typeOf(d).TO
+    if d.lookup_function_no == FUNC_BYTE:
+        DICT.lookup_family.byte_insert_clean_function(d, hash, i)
+    elif d.lookup_function_no == FUNC_SHORT:
+        DICT.lookup_family.short_insert_clean_function(d, hash, i)
+    elif IS_64BIT and d.lookup_function_no == FUNC_INT:
+        DICT.lookup_family.int_insert_clean_function(d, hash, i)
+    elif d.lookup_function_no == FUNC_LONG:
+        DICT.lookup_family.long_insert_clean_function(d, hash, i)
+    else:
+        assert False
+
+def ll_valid_from_flag(entries, i):
+    return entries[i].f_valid
+
+def ll_valid_from_key(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    return entries[i].key != dummy
+
+def ll_valid_from_value(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    return entries[i].value != dummy
+
+def ll_mark_deleted_in_flag(entries, i):
+    entries[i].f_valid = False
+
+def ll_mark_deleted_in_key(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    entries[i].key = dummy
+
+def ll_mark_deleted_in_value(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    entries[i].value = dummy
+
+def ll_hash_from_cache(entries, i):
+    return entries[i].f_hash
+
+def ll_hash_recomputed(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    return ENTRIES.fasthashfn(entries[i].key)
+
+def ll_keyhash_custom(d, key):
+    DICT = lltype.typeOf(d).TO
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+
+def ll_keyeq_custom(d, key1, key2):
+    DICT = lltype.typeOf(d).TO
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+
+def ll_dict_len(d):
+    return d.num_items
+
+def ll_dict_bool(d):
+    # check if a dict is True, allowing for None
+    return bool(d) and d.num_items != 0
+
+def ll_dict_getitem(d, key):
+    index = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
+    if index != -1:
+        return d.entries[index].value
+    else:
+        raise KeyError
+
+def ll_dict_setitem(d, key, value):
+    hash = d.keyhash(key)
+    index = d.lookup_function(d, key, hash, FLAG_STORE)
+    return _ll_dict_setitem_lookup_done(d, key, value, hash, index)
+
+# It may be safe to look inside always, it has a few branches though, and their
+# frequencies needs to be investigated.
+ at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
+def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    if i >= 0:
+        entry = d.entries[i]
+        entry.value = value
+    else:
+        if len(d.entries) == d.num_used_items:
+            if ll_dict_grow(d):
+                ll_call_insert_clean_function(d, hash, d.num_used_items)
+        entry = d.entries[d.num_used_items]
+        entry.key = key
+        entry.value = value
+        if hasattr(ENTRY, 'f_hash'):
+            entry.f_hash = hash
+        if hasattr(ENTRY, 'f_valid'):
+            entry.f_valid = True
+        d.num_used_items += 1
+        d.num_items += 1
+        rc = d.resize_counter - 3
+        if rc <= 0:
+            ll_dict_resize(d)
+            rc = d.resize_counter - 3
+            ll_assert(rc > 0, "ll_dict_resize failed?")
+        d.resize_counter = rc
+
+def _ll_dict_insertclean(d, key, value, hash):
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    ll_call_insert_clean_function(d, hash, d.num_used_items)
+    entry = d.entries[d.num_used_items]
+    entry.key = key
+    entry.value = value
+    if hasattr(ENTRY, 'f_hash'):
+        entry.f_hash = hash
+    if hasattr(ENTRY, 'f_valid'):
+        entry.f_valid = True
+    d.num_used_items += 1
+    d.num_items += 1
+    rc = d.resize_counter - 3
+    d.resize_counter = rc
+
+def _ll_len_of_d_indexes(d):
+    # xxx Haaaack: returns len(d.indexes).  Works independently of
+    # the exact type pointed to by d, using a forced cast...
+    return len(rffi.cast(DICTINDEX_BYTE, d.indexes))
+
+def _overallocate_entries_len(baselen):
+    # This over-allocates proportional to the list size, making room
+    # for additional growth.  The over-allocation is mild, but is
+    # enough to give linear-time amortized behavior over a long
+    # sequence of appends() in the presence of a poorly-performing
+    # system malloc().
+    # The growth pattern is:  0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
+    newsize = baselen + 1
+    if newsize < 9:
+        some = 3
+    else:
+        some = 6
+    some += newsize >> 3
+    return newsize + some
+
+ at jit.dont_look_inside
+def ll_dict_grow(d):
+    if d.num_items < d.num_used_items // 4:
+        ll_dict_remove_deleted_items(d)
+        return True
+
+    new_allocated = _overallocate_entries_len(len(d.entries))
+
+    # Detect an obscure case where the indexes numeric type is too
+    # small to store all the entry indexes
+    if (max(128, _ll_len_of_d_indexes(d)) - new_allocated
+                   < MIN_INDEXES_MINUS_ENTRIES):
+        ll_dict_remove_deleted_items(d)
+        return True
+
+    newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+    rgc.ll_arraycopy(d.entries, newitems, 0, 0, len(d.entries))
+    d.entries = newitems
+    return False
+
+def ll_dict_remove_deleted_items(d):
+    new_allocated = _overallocate_entries_len(d.num_items)
+    if new_allocated < len(d.entries) // 2:
+        newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+    else:
+        newitems = d.entries
+    #
+    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
+    isrc = 0
+    idst = 0
+    while isrc < len(d.entries):
+        if d.entries.valid(isrc):
+            src = d.entries[isrc]
+            dst = newitems[idst]
+            dst.key = src.key
+            dst.value = src.value
+            if hasattr(ENTRY, 'f_hash'):
+                dst.f_hash = src.f_hash
+            if hasattr(ENTRY, 'f_valid'):
+                assert src.f_valid
+                dst.f_valid = True
+            idst += 1
+        isrc += 1
+    d.entries = newitems
+    assert d.num_items == idst
+    d.num_used_items = idst
+
+    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+
+
+def ll_dict_delitem(d, key):
+    index = d.lookup_function(d, key, d.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        raise KeyError
+    _ll_dict_del(d, index)
+
+ at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
+def _ll_dict_del(d, index):
+    d.entries.mark_deleted(index)
+    d.num_items -= 1
+    # clear the key and the value if they are GC pointers
+    ENTRIES = lltype.typeOf(d.entries).TO
+    ENTRY = ENTRIES.OF
+    entry = d.entries[index]
+    if ENTRIES.must_clear_key:
+        entry.key = lltype.nullptr(ENTRY.key.TO)
+    if ENTRIES.must_clear_value:
+        entry.value = lltype.nullptr(ENTRY.value.TO)
+    #
+    # The rest is commented out: like CPython we no longer shrink the
+    # dictionary here.  It may shrink later if we try to append a number
+    # of new items to it.  Unsure if this behavior was designed in
+    # CPython or is accidental.  A design reason would be that if you
+    # delete all items in a dictionary (e.g. with a series of
+    # popitem()), then CPython avoids shrinking the table several times.
+    #num_entries = len(d.entries)
+    #if num_entries > DICT_INITSIZE and d.num_items <= num_entries / 4:
+    #    ll_dict_resize(d)
+    # A previous xxx: move the size checking and resize into a single
+    # call which is opaque to the JIT when the dict isn't virtual, to
+    # avoid extra branches.
+
+def ll_dict_resize(d):
+    # make a 'new_size' estimate and shrink it if there are many
+    # deleted entry markers.  See CPython for why it is a good idea to
+    # quadruple the dictionary size as long as it's not too big.
+    num_items = d.num_items
+    if num_items > 50000:
+        new_estimate = num_items * 2
+    else:
+        new_estimate = num_items * 4
+    new_size = DICT_INITSIZE
+    while new_size <= new_estimate:
+        new_size *= 2
+
+    if new_size < _ll_len_of_d_indexes(d):
+        ll_dict_remove_deleted_items(d)
+    else:
+        ll_dict_reindex(d, new_size)
+ll_dict_resize.oopspec = 'dict.resize(d)'
+
+def ll_dict_reindex(d, new_size):
+    ll_malloc_indexes_and_choose_lookup(d, new_size)
+    d.resize_counter = new_size * 2 - d.num_items * 3
+    assert d.resize_counter > 0
+    #
+    entries = d.entries
+    i = 0
+    while i < d.num_used_items:
+        if entries.valid(i):
+            hash = entries.hash(i)
+            ll_call_insert_clean_function(d, hash, i)
+        i += 1
+    #old_entries.delete() XXXX!
+
+# ------- a port of CPython's dictobject.c's lookdict implementation -------
+PERTURB_SHIFT = 5
+
+FREE = 0
+DELETED = 1
+VALID_OFFSET = 2
+MIN_INDEXES_MINUS_ENTRIES = VALID_OFFSET + 1
+
+FLAG_LOOKUP = 0
+FLAG_STORE = 1
+FLAG_DELETE = 2
+FLAG_DELETE_TRY_HARD = 3
+
+def new_lookup_functions(LOOKUP_FUNC, STORECLEAN_FUNC, T, rtyper=None):
+    INDEXES = lltype.Ptr(lltype.GcArray(T))
+
+    def ll_kill_something(d):
+        i = 0
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        while True:
+            index = rffi.cast(lltype.Signed, indexes[i])
+            if index >= VALID_OFFSET:
+                indexes[i] = rffi.cast(T, DELETED)
+                return index
+            i += 1
+
+    @jit.look_inside_iff(lambda d, key, hash, store_flag:
+                         jit.isvirtual(d) and jit.isconstant(key))
+    def ll_dict_lookup(d, key, hash, store_flag):
+        entries = d.entries
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        mask = len(indexes) - 1
+        i = r_uint(hash & mask)
+        # do the first try before any looping
+        ENTRIES = lltype.typeOf(entries).TO
+        direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
+        index = rffi.cast(lltype.Signed, indexes[intmask(i)])
+        if index >= VALID_OFFSET:
+            checkingkey = entries[index - VALID_OFFSET].key
+            if direct_compare and checkingkey == key:
+                if store_flag == FLAG_DELETE:
+                    indexes[i] = rffi.cast(T, DELETED)
+                return index - VALID_OFFSET   # found the entry
+            if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
+                # correct hash, maybe the key is e.g. a different pointer to
+                # an equal object
+                found = d.keyeq(checkingkey, key)
+                #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found)
+                if d.paranoia:
+                    if (entries != d.entries or lltype.cast_opaque_ptr(llmemory.GCREF, indexes) != d.indexes or
+                        not entries.valid(index - VALID_OFFSET) or
+                        entries[index - VALID_OFFSET].key != checkingkey):
+                        # the compare did major nasty stuff to the dict: start over
+                        return ll_dict_lookup(d, key, hash, store_flag)
+                if found:
+                    if store_flag == FLAG_DELETE:
+                        indexes[i] = rffi.cast(T, DELETED)
+                    return index - VALID_OFFSET
+            deletedslot = -1
+        elif index == DELETED:
+            deletedslot = intmask(i)
+        else:
+            # pristine entry -- lookup failed
+            if store_flag == FLAG_STORE:
+                indexes[i] = rffi.cast(T, d.num_used_items + VALID_OFFSET)
+            elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
+                return ll_kill_something(d)
+            return -1
+
+        # In the loop, a deleted entry (everused and not valid) is by far
+        # (factor of 100s) the least likely outcome, so test for that last.
+        perturb = r_uint(hash)
+        while 1:
+            # compute the next index using unsigned arithmetic
+            i = (i << 2) + i + perturb + 1
+            i = i & mask
+            index = rffi.cast(lltype.Signed, indexes[intmask(i)])
+            if index == FREE:
+                if store_flag == FLAG_STORE:
+                    if deletedslot == -1:
+                        deletedslot = intmask(i)
+                    indexes[deletedslot] = rffi.cast(T, d.num_used_items +
+                                                     VALID_OFFSET)
+                elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
+                    return ll_kill_something(d)
+                return -1
+            elif index >= VALID_OFFSET:
+                checkingkey = entries[index - VALID_OFFSET].key
+                if direct_compare and checkingkey == key:
+                    if store_flag == FLAG_DELETE:
+                        indexes[i] = rffi.cast(T, DELETED)
+                    return index - VALID_OFFSET   # found the entry
+                if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
+                    # correct hash, maybe the key is e.g. a different pointer to
+                    # an equal object
+                    found = d.keyeq(checkingkey, key)
+                    if d.paranoia:
+                        if (entries != d.entries or lltype.cast_opaque_ptr(llmemory.GCREF, indexes) != d.indexes or
+                            not entries.valid(index - VALID_OFFSET) or
+                            entries[index - VALID_OFFSET].key != checkingkey):
+                            # the compare did major nasty stuff to the dict: start over
+                            return ll_dict_lookup(d, key, hash, store_flag)
+                    if found:
+                        if store_flag == FLAG_DELETE:
+                            indexes[i] = rffi.cast(T, DELETED)
+                        return index - VALID_OFFSET
+            elif deletedslot == -1:
+                deletedslot = intmask(i)
+            perturb >>= PERTURB_SHIFT
+
+    def ll_dict_store_clean(d, hash, index):
+        # a simplified version of ll_dict_lookup() which assumes that the
+        # key is new, and the dictionary doesn't contain deleted entries.
+        # It only finds the next free slot for the given hash.
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        mask = len(indexes) - 1
+        i = r_uint(hash & mask)
+        perturb = r_uint(hash)
+        while rffi.cast(lltype.Signed, indexes[i]) != 0:
+            i = (i << 2) + i + perturb + 1
+            i = i & mask
+            perturb >>= PERTURB_SHIFT
+        indexes[i] = rffi.cast(T, index + VALID_OFFSET)
+
+    return (llhelper_or_compile(rtyper, LOOKUP_FUNC, ll_dict_lookup),
+            llhelper_or_compile(rtyper, STORECLEAN_FUNC, ll_dict_store_clean))
+
+# ____________________________________________________________
+#
+#  Irregular operations.
+
+DICT_INITSIZE = 8
+
+def ll_newdict(DICT):
+    d = DICT.allocate()
+    d.entries = DICT.lookup_family.empty_array
+    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = DICT_INITSIZE * 2
+    return d
+
+def ll_newdict_size(DICT, orig_length_estimate):
+    length_estimate = (orig_length_estimate // 2) * 3
+    n = DICT_INITSIZE
+    while n < length_estimate:
+        n *= 2
+    d = DICT.allocate()
+    d.entries = DICT.entries.TO.allocate(orig_length_estimate)
+    ll_malloc_indexes_and_choose_lookup(d, n)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = n * 2
+    return d
+
+# rpython.memory.lldict uses a dict based on Struct and Array
+# instead of GcStruct and GcArray, which is done by using different
+# 'allocate' and 'delete' adtmethod implementations than the ones below
+def _ll_malloc_dict(DICT):
+    return lltype.malloc(DICT)
+def _ll_malloc_entries(ENTRIES, n):
+    return lltype.malloc(ENTRIES, n, zero=True)
+def _ll_free_entries(entries):
+    pass
+
+
+def rtype_r_dict(hop):
+    r_dict = hop.r_result
+    if not r_dict.custom_eq_hash:
+        raise TyperError("r_dict() call does not return an r_dict instance")
+    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
+    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
+    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
+    hop.exception_cannot_occur()
+    v_result = hop.gendirectcall(ll_newdict, cDICT)
+    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
+        hop.genop('setfield', [v_result, cname, v_eqfn])
+    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
+        hop.genop('setfield', [v_result, cname, v_hashfn])
+    return v_result
+
+# ____________________________________________________________
+#
+#  Iteration.
+
+def get_ll_dictiter(DICTPTR):
+    return lltype.Ptr(lltype.GcStruct('dictiter',
+                                      ('dict', DICTPTR),
+                                      ('index', lltype.Signed)))
+
+class DictIteratorRepr(AbstractDictIteratorRepr):
+
+    def __init__(self, r_dict, variant="keys"):
+        self.r_dict = r_dict
+        self.variant = variant
+        self.lowleveltype = get_ll_dictiter(r_dict.lowleveltype)
+        self.ll_dictiter = ll_dictiter
+        self.ll_dictnext = ll_dictnext_group[variant]
+
+
+def ll_dictiter(ITERPTR, d):
+    iter = lltype.malloc(ITERPTR.TO)
+    iter.dict = d
+    iter.index = 0
+    return iter
+
+def _make_ll_dictnext(kind):
+    # make three versions of the following function: keys, values, items
+    @jit.look_inside_iff(lambda RETURNTYPE, iter: jit.isvirtual(iter)
+                         and (iter.dict is None or
+                              jit.isvirtual(iter.dict)))
+    @jit.oopspec("dictiter.next%s(iter)" % kind)
+    def ll_dictnext(RETURNTYPE, iter):
+        # note that RETURNTYPE is None for keys and values
+        dict = iter.dict
+        if not dict:
+            raise StopIteration
+
+        entries = dict.entries
+        index = iter.index
+        assert index >= 0
+        entries_len = dict.num_used_items
+        while index < entries_len:
+            entry = entries[index]
+            is_valid = entries.valid(index)
+            index = index + 1
+            if is_valid:
+                iter.index = index
+                if RETURNTYPE is lltype.Void:
+                    return None
+                elif kind == 'items':
+                    r = lltype.malloc(RETURNTYPE.TO)
+                    r.item0 = recast(RETURNTYPE.TO.item0, entry.key)
+                    r.item1 = recast(RETURNTYPE.TO.item1, entry.value)
+                    return r
+                elif kind == 'keys':
+                    return entry.key
+                elif kind == 'values':
+                    return entry.value
+
+        # clear the reference to the dict and prevent restarts
+        iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
+        raise StopIteration
+
+    return ll_dictnext
+
+ll_dictnext_group = {'keys'  : _make_ll_dictnext('keys'),
+                     'values': _make_ll_dictnext('values'),
+                     'items' : _make_ll_dictnext('items')}
+
+# _____________________________________________________________
+# methods
+
+def ll_dict_get(dict, key, default):
+    index = dict.lookup_function(dict, key, dict.keyhash(key), FLAG_LOOKUP)
+    if index == -1:
+        return default
+    else:
+        return dict.entries[index].value
+
+def ll_dict_setdefault(dict, key, default):
+    hash = dict.keyhash(key)
+    index = dict.lookup_function(dict, key, hash, FLAG_STORE)
+    if index == -1:
+        _ll_dict_setitem_lookup_done(dict, key, default, hash, -1)
+        return default
+    else:
+        return dict.entries[index].value
+
+def ll_dict_copy(dict):
+    DICT = lltype.typeOf(dict).TO
+    newdict = DICT.allocate()
+    newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
+
+    newdict.num_items = dict.num_items
+    newdict.num_used_items = dict.num_used_items
+    if hasattr(DICT, 'fnkeyeq'):
+        newdict.fnkeyeq = dict.fnkeyeq
+    if hasattr(DICT, 'fnkeyhash'):
+        newdict.fnkeyhash = dict.fnkeyhash
+
+    i = 0
+    while i < newdict.num_used_items:
+        d_entry = newdict.entries[i]
+        entry = dict.entries[i]
+        ENTRY = lltype.typeOf(newdict.entries).TO.OF
+        d_entry.key = entry.key
+        if hasattr(ENTRY, 'f_valid'):
+            d_entry.f_valid = entry.f_valid
+        d_entry.value = entry.value
+        if hasattr(ENTRY, 'f_hash'):
+            d_entry.f_hash = entry.f_hash
+        i += 1
+
+    ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
+    return newdict
+ll_dict_copy.oopspec = 'dict.copy(dict)'
+
+def ll_dict_clear(d):
+    if d.num_used_items == 0:
+        return
+    DICT = lltype.typeOf(d).TO
+    old_entries = d.entries
+    d.entries = DICT.lookup_family.empty_array
+    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = DICT_INITSIZE * 2
+    # old_entries.delete() XXX
+ll_dict_clear.oopspec = 'dict.clear(d)'
+
+def ll_dict_update(dic1, dic2):
+    i = 0
+    while i < dic2.num_used_items:
+        entries = dic2.entries
+        if entries.valid(i):
+            entry = entries[i]
+            hash = entries.hash(i)
+            key = entry.key
+            value = entry.value
+            index = dic1.lookup_function(dic1, key, hash, FLAG_STORE)
+            _ll_dict_setitem_lookup_done(dic1, key, value, hash, index)
+        i += 1
+ll_dict_update.oopspec = 'dict.update(dic1, dic2)'
+
+# this is an implementation of keys(), values() and items()
+# in a single function.
+# note that by specialization on func, three different
+# and very efficient functions are created.
+
+def recast(P, v):
+    if isinstance(P, lltype.Ptr):
+        return lltype.cast_pointer(P, v)
+    else:
+        return v
+
+def _make_ll_keys_values_items(kind):
+    def ll_kvi(LIST, dic):
+        res = LIST.ll_newlist(dic.num_items)
+        entries = dic.entries
+        dlen = dic.num_used_items
+        items = res.ll_items()
+        i = 0
+        p = 0
+        while i < dlen:
+            if entries.valid(i):
+                ELEM = lltype.typeOf(items).TO.OF
+                if ELEM is not lltype.Void:
+                    entry = entries[i]
+                    if kind == 'items':
+                        r = lltype.malloc(ELEM.TO)
+                        r.item0 = recast(ELEM.TO.item0, entry.key)
+                        r.item1 = recast(ELEM.TO.item1, entry.value)
+                        items[p] = r
+                    elif kind == 'keys':
+                        items[p] = recast(ELEM, entry.key)
+                    elif kind == 'values':
+                        items[p] = recast(ELEM, entry.value)
+                p += 1
+            i += 1
+        assert p == res.ll_length()
+        return res
+    ll_kvi.oopspec = 'dict.%s(dic)' % kind
+    return ll_kvi
+
+ll_dict_keys   = _make_ll_keys_values_items('keys')
+ll_dict_values = _make_ll_keys_values_items('values')
+ll_dict_items  = _make_ll_keys_values_items('items')
+
+def ll_dict_contains(d, key):
+    i = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
+    return i != -1
+
+def _ll_getnextitem(dic):
+    if dic.num_items == 0:
+        raise KeyError
+
+    entries = dic.entries
+
+    while True:
+        i = dic.num_used_items - 1
+        if entries.valid(i):
+            break
+        dic.num_used_items -= 1
+
+    key = entries[i].key
+    index = dic.lookup_function(dic, key, entries.hash(i),
+                                FLAG_DELETE_TRY_HARD)
+    # if the lookup function returned me a random strange thing,
+    # don't care about deleting the item
+    if index == dic.num_used_items - 1:
+        dic.num_used_items -= 1
+    else:
+        assert index != -1
+    return index
+
+def ll_dict_popitem(ELEM, dic):
+    i = _ll_getnextitem(dic)
+    entry = dic.entries[i]
+    r = lltype.malloc(ELEM.TO)
+    r.item0 = recast(ELEM.TO.item0, entry.key)
+    r.item1 = recast(ELEM.TO.item1, entry.value)
+    _ll_dict_del(dic, i)
+    return r
+
+def ll_dict_pop(dic, key):
+    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        raise KeyError
+    value = dic.entries[index].value
+    _ll_dict_del(dic, index)
+    return value
+
+def ll_dict_pop_default(dic, key, dfl):
+    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        return dfl
+    value = dic.entries[index].value
+    _ll_dict_del(dic, index)
+    return value
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -1,3 +1,5 @@
+from collections import OrderedDict
+
 from rpython.annotator import model as annmodel
 from rpython.flowspace.model import Constant
 from rpython.rlib import rarithmetic, objectmodel
@@ -726,10 +728,29 @@
 
     raise TyperError("hasattr is only suported on a constant")
 
+def rtype_ordered_dict(hop):
+    from rpython.rtyper.lltypesystem.rordereddict import ll_newdict
+
+    hop.exception_cannot_occur()
+    r_dict = hop.r_result
+    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
+    v_result = hop.gendirectcall(ll_newdict, cDICT)
+    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
+    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
+    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
+        hop.genop('setfield', [v_result, cname, v_eqfn])
+    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
+        hop.genop('setfield', [v_result, cname, v_hashfn])
+    return v_result
+
 BUILTIN_TYPER[objectmodel.instantiate] = rtype_instantiate
 BUILTIN_TYPER[isinstance] = rtype_builtin_isinstance
 BUILTIN_TYPER[hasattr] = rtype_builtin_hasattr
 BUILTIN_TYPER[objectmodel.r_dict] = rtype_r_dict
+BUILTIN_TYPER[OrderedDict] = rtype_ordered_dict
+BUILTIN_TYPER[objectmodel.r_ordereddict] = rtype_ordered_dict
 
 # _________________________________________________________________
 # weakrefs
diff --git a/rpython/rtyper/rdict.py b/rpython/rtyper/rdict.py
--- a/rpython/rtyper/rdict.py
+++ b/rpython/rtyper/rdict.py
@@ -4,8 +4,12 @@
 
 
 class __extend__(annmodel.SomeDict):
+    def get_dict_repr(self):
+        from rpython.rtyper.lltypesystem.rdict import DictRepr
+
+        return DictRepr
+
     def rtyper_makerepr(self, rtyper):
-        from rpython.rtyper.lltypesystem.rdict import DictRepr
         dictkey = self.dictdef.dictkey
         dictvalue = self.dictdef.dictvalue
         s_key = dictkey.s_value
@@ -16,7 +20,7 @@
                                       rtyper.getrepr(dictkey.s_rdict_hashfn))
         else:
             custom_eq_hash = None
-        return DictRepr(rtyper, lambda: rtyper.getrepr(s_key),
+        return self.get_dict_repr()(rtyper, lambda: rtyper.getrepr(s_key),
                         lambda: rtyper.getrepr(s_value), dictkey, dictvalue,
                         custom_eq_hash, force_non_null)
 
@@ -25,6 +29,11 @@
         self.dictdef.dictvalue.dont_change_any_more = True
         return (self.__class__, self.dictdef.dictkey, self.dictdef.dictvalue)
 
+class __extend__(annmodel.SomeOrderedDict):
+    def get_dict_repr(self):
+        from rpython.rtyper.lltypesystem.rordereddict import OrderedDictRepr
+
+        return OrderedDictRepr
 
 class AbstractDictRepr(rmodel.Repr):
 
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -22,11 +22,11 @@
         yield x
 
 
-class TestRdict(BaseRtypingTest):
-
+class BaseTestRDict(BaseRtypingTest):
     def test_dict_creation(self):
         def createdict(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             return d['hello']
 
         res = self.interpret(createdict, [42])
@@ -34,7 +34,8 @@
 
     def test_dict_getitem_setitem(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             d['world'] = i + 1
             return d['hello'] * d['world']
         res = self.interpret(func, [6])
@@ -42,7 +43,8 @@
 
     def test_dict_getitem_keyerror(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             try:
                 return d['world']
             except KeyError:
@@ -52,7 +54,8 @@
 
     def test_dict_del_simple(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             d['world'] = i + 1
             del d['hello']
             return len(d)
@@ -61,7 +64,8 @@
 
     def test_dict_clear(self):
         def func(i):
-            d = {'abc': i}
+            d = self.newdict()
+            d['abc'] = i
             d['def'] = i+1
             d.clear()
             d['ghi'] = i+2
@@ -72,7 +76,8 @@
 
     def test_empty_strings(self):
         def func(i):
-            d = {'' : i}
+            d = self.newdict()
+            d[''] = i
             del d['']
             try:
                 d['']
@@ -84,7 +89,8 @@
         assert res == 1
 
         def func(i):
-            d = {'' : i}
+            d = self.newdict()
+            d[''] = i
             del d['']
             d[''] = i + 1
             return len(d)
@@ -94,9 +100,10 @@
     def test_dict_bool(self):
         def func(i):
             if i:
-                d = {}
+                d = self.newdict()
             else:
-                d = {i: i+1}
+                d = self.newdict()
+                d[i] = i+1
             if d:
                 return i
             else:
@@ -106,17 +113,20 @@
 
     def test_contains(self):
         def func(x, y):
-            d = {x: x+1}
+            d = self.newdict()
+            d[x] = x+1
             return y in d
         assert self.interpret(func, [42, 0]) == False
         assert self.interpret(func, [42, 42]) == True
 
     def test_contains_2(self):
-        d = {'5': None, '7': None}
+        d = self.newdict()
+        d['5'] = None
+        d['7'] = None
         def func(x):
             return chr(x) in d
-        #assert self.interpret(func, [ord('5')]) == True
-        #assert self.interpret(func, [ord('6')]) == False
+        assert self.interpret(func, [ord('5')]) == True
+        assert self.interpret(func, [ord('6')]) == False
 
         def func(n):
             return str(n) in d
@@ -124,7 +134,7 @@
 
     def test_dict_iteration(self):
         def func(i, j):
-            d = {}
+            d = self.newdict()
             d['hello'] = i
             d['world'] = j
             k = 1
@@ -136,7 +146,7 @@
 
     def test_dict_itermethods(self):
         def func():
-            d = {}
+            d = self.newdict()
             d['hello'] = 6
             d['world'] = 7
             k1 = k2 = k3 = 1
@@ -151,19 +161,9 @@
         res = self.interpret(func, [])
         assert res == 42 + 42 + 42
 
-    def test_two_dicts_with_different_value_types(self):
-        def func(i):
-            d1 = {}
-            d1['hello'] = i + 1
-            d2 = {}
-            d2['world'] = d1
-            return d2['world']['hello']
-        res = self.interpret(func, [5])
-        assert res == 6
-
     def test_dict_get(self):
         def func():
-            dic = {}
+            dic = self.newdict()
             x1 = dic.get('hi', 42)
             dic['blah'] = 1 # XXX this triggers type determination
             x2 = dic.get('blah', 2)
@@ -174,7 +174,7 @@
     def test_dict_get_empty(self):
         def func():
             # this time without writing to the dict
-            dic = {}
+            dic = self.newdict()
             x1 = dic.get('hi', 42)
             x2 = dic.get('blah', 2)
             return x1 * 10 + x2
@@ -183,14 +183,14 @@
 
     def test_dict_setdefault(self):
         def f():
-            d = {}
+            d = self.newdict()
             d.setdefault('a', 2)
             return d['a']
         res = self.interpret(f, ())
         assert res == 2
 
         def f():
-            d = {}
+            d = self.newdict()
             d.setdefault('a', 2)
             x = d.setdefault('a', -3)
             return x
@@ -200,7 +200,9 @@
     def test_dict_copy(self):
         def func():
             # XXX this does not work if we use chars, only!
-            dic = {'ab':1, 'b':2}
+            dic = self.newdict()
+            dic['ab'] = 1
+            dic['b'] = 2
             d2 = dic.copy()
             ok = 1
             for key in d2:
@@ -215,8 +217,12 @@
 
     def test_dict_update(self):
         def func():
-            dic = {'ab':1000, 'b':200}
-            d2 = {'b':30, 'cb':4}
+            dic = self.newdict()
+            dic['ab'] = 1000
+            dic['b'] = 200
+            d2 = self.newdict()
+            d2['b'] = 30
+            d2['cb'] = 4
             dic.update(d2)
             ok = len(dic) == 3
             sum = ok
@@ -228,7 +234,9 @@
 
     def test_dict_keys(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             keys = dic.keys()
             return ord(keys[0][1]) + ord(keys[1][1]) - 2*ord('0') + len(keys)
         res = self.interpret(func, ())#, view=True)
@@ -240,8 +248,11 @@
         class A(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {A(): 1, A(): 2}
+            dic0 = self.newdict()
+            dic0[Empty()] = 2
+            dic = self.newdict()
+            dic[A()] = 1
+            dic[A()] = 2
             keys = dic.keys()
             return (isinstance(keys[1], A))*2+(isinstance(keys[0],A))
         res = self.interpret(func, [])
@@ -253,8 +264,11 @@
         class A(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {A(): 1, A(): 2}
+            dic0 = self.newdict()
+            dic0[Empty()] = 2
+            dic = self.newdict()
+            dic[A()] = 1
+            dic[A()] = 2
             a = 0
             for k in dic.iterkeys():
                 a += isinstance(k, A)
@@ -264,7 +278,9 @@
 
     def test_dict_values(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             values = dic.values()
             return values[0] + values[1] + len(values)
         res = self.interpret(func, ())
@@ -274,7 +290,9 @@
         class A:
             pass
         def func():
-            dic = {1: A(), 2: A()}
+            dic = self.newdict()
+            dic[1] = A()
+            dic[2] = A()
             vals = dic.values()
             return (isinstance(vals[1], A))*2+(isinstance(vals[0],A))
         res = self.interpret(func, [])
@@ -284,7 +302,9 @@
         class A:
             pass
         def func():
-            dic = {1: A(), 2: A()}
+            dic = self.newdict()
+            dic[1] = A()
+            dic[2] = A()
             a = 0
             for v in dic.itervalues():
                 a += isinstance(v, A)
@@ -300,8 +320,11 @@
         class B(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {B(): A(), B(): A()}
+            dic0 = self.newdict()
+            dic0[Empty()] = A()
+            dic = self.newdict()
+            dic[B()] = A()
+            dic[B()] = A()
             items = dic.items()
             b = 0
             a = 0
@@ -320,8 +343,11 @@
         class B(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {B(): A(), B(): A()}
+            dic0 = self.newdict()
+            dic0[Empty()] = A()
+            dic = self.newdict()
+            dic[B()] = A()
+            dic[B()] = A()
             b = 0
             a = 0
             for k, v in dic.iteritems():
@@ -333,7 +359,9 @@
 
     def test_dict_items(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             items = dic.items()
             res = len(items)
             for key, value in items:
@@ -344,13 +372,17 @@
 
     def test_dict_contains(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             return ' 4' in dic and ' 9' not in dic
         res = self.interpret(func, ())
         assert res is True
 
     def test_dict_contains_with_constant_dict(self):
-        dic = {'4':1000, ' 8':200}
+        dic = self.newdict()
+        dic['4'] = 1000
+        dic['8'] = 200
         def func(i):
             return chr(i) in dic
         res = self.interpret(func, [ord('4')])
@@ -367,7 +399,9 @@
             a = A()
             a.d = None
             if n > 0:
-                a.d = {str(n): 1, "42": 2}
+                a.d = self.newdict()
+                a.d[str(n)] = 1
+                a.d["42"] = 2
                 del a.d["42"]
             return negate(a.d)
         res = self.interpret(func, [10])
@@ -379,7 +413,8 @@
 
     def test_int_dict(self):
         def func(a, b):
-            dic = {12: 34}
+            dic = self.newdict()
+            dic[12] = 34
             dic[a] = 1000
             return dic.get(b, -123)
         res = self.interpret(func, [12, 12])
@@ -403,7 +438,7 @@
         def f():
             a = A()
             b = B()
-            d = {}
+            d = self.newdict()
             d[b] = 7
             d[a] = 3
             return len(d) + d[a] + d[b]
@@ -411,7 +446,9 @@
         assert res == 12
 
     def test_captured_get(self):
-        get = {1:2}.get
+        d = self.newdict()
+        d[1] = 2
+        get = d.get
         def f():
             return get(1, 3)+get(2, 4)
         res = self.interpret(f, [])
@@ -431,40 +468,21 @@
         def f():
             lst = [A()]
             res1 = A() in lst
-            d2 = {B(): None, B(): None}
+            d2 = self.newdict()
+            d2[B()] = None
+            d2[B()] = None
             return res1+len(d2)
         res = self.interpret(f, [])
         assert res == 2
 
-
-    def test_type_erase(self):
-        class A(object):
-            pass
-        class B(object):
-            pass
-
-        def f():
-            return {A(): B()}, {B(): A()}
-
-        t = TranslationContext()
-        s = t.buildannotator().build_types(f, [])
-        rtyper = t.buildrtyper()
-        rtyper.specialize()
-
-        s_AB_dic = s.items[0]
-        s_BA_dic = s.items[1]
-
-        r_AB_dic = rtyper.getrepr(s_AB_dic)
-        r_BA_dic = rtyper.getrepr(s_AB_dic)
-
-        assert r_AB_dic.lowleveltype == r_BA_dic.lowleveltype
-
     def test_identity_hash_is_fast(self):
         class A(object):
             pass
 
         def f():
-            return {A(): 1}
+            d = self.newdict()
+            d[A()] = 1
+            return d
 
         t = TranslationContext()
         s = t.buildannotator().build_types(f, [])
@@ -476,7 +494,7 @@
 
     def test_tuple_dict(self):
         def f(i):
-            d = {}
+            d = self.newdict()
             d[(1, 4.5, (str(i), 2), 2)] = 4
             d[(1, 4.5, (str(i), 2), 3)] = 6
             return d[(1, 4.5, (str(i), 2), i)]
@@ -486,9 +504,9 @@
 
     def test_dict_of_dict(self):
         def f(n):
-            d = {}
+            d = self.newdict()
             d[5] = d
-            d[6] = {}
+            d[6] = self.newdict()
             return len(d[n])
 
         res = self.interpret(f, [5])
@@ -504,10 +522,9 @@
             pass
 
         def f(i):
-            d = {
-                A: 3,
-                B: 4,
-            }
+            d = self.newdict()
+            d[A] = 3
+            d[B] = 4
             if i:
                 cls = A
             else:
@@ -526,7 +543,9 @@
         class B(A):
             pass
 
-        d = {(A, 3): 3, (B, 0): 4}
+        d = self.newdict()
+        d[(A, 3)] = 3
+        d[(B, 0)] = 4
 
         def f(i):
             if i:
@@ -553,7 +572,9 @@
                 return 42
             return -1
         def g(n):
-            d = {1: n, 2: 2*n}
+            d = self.newdict()
+            d[1] = n
+            d[2] = 2*n
             return f(d)
         res = self.interpret(g, [3])
         assert res == 6
@@ -566,51 +587,19 @@
                 return 42
             return -1
         def g(n):
-            d = {1: n}
+            d = self.newdict()
+            d[1] = n
             f(d)
             return d[2]
         res = self.interpret(g, [3])
         assert res == 77
 
-    def test_r_dict(self):
-        class FooError(Exception):
-            pass
-        def myeq(n, m):
-            return n == m
-        def myhash(n):
-            if n < 0:
-                raise FooError
-            return -n
-        def f(n):
-            d = r_dict(myeq, myhash)
-            for i in range(10):
-                d[i] = i*i
-            try:
-                value1 = d[n]
-            except FooError:
-                value1 = 99
-            try:
-                value2 = n in d
-            except FooError:
-                value2 = 99
-            try:
-                value3 = d[-n]
-            except FooError:
-                value3 = 99
-            try:
-                value4 = (-n) in d
-            except FooError:
-                value4 = 99
-            return (value1 * 1000000 +
-                    value2 * 10000 +
-                    value3 * 100 +
-                    value4)
-        res = self.interpret(f, [5])
-        assert res == 25019999
-
     def test_resize_during_iteration(self):
         def func():
-            d = {5: 1, 6: 2, 7: 3}
+            d = self.newdict()
+            d[5] = 1
+            d[6] = 2
+            d[7] = 3
             try:
                 for key, value in d.iteritems():
                     d[key^16] = value*2
@@ -625,14 +614,21 @@
 
     def test_change_during_iteration(self):
         def func():
-            d = {'a': 1, 'b': 2}
+            d = self.newdict()
+            d['a'] = 1
+            d['b'] = 2
             for key in d:
                 d[key] = 42
             return d['a']
         assert self.interpret(func, []) == 42
 
     def test_dict_of_floats(self):
-        d = {3.0: 42, 3.1: 43, 3.2: 44, 3.3: 45, 3.4: 46}
+        d = self.newdict()
+        d[3.0] = 42
+        d[3.1] = 43
+        d[3.2] = 44
+        d[3.3] = 45
+        d[3.4] = 46
         def fn(f):
             return d[f]
 
@@ -643,7 +639,9 @@
         for r_t in [r_uint, r_longlong, r_ulonglong]:
             if r_t is r_int:
                 continue    # for 64-bit platforms: skip r_longlong
-            d = {r_t(2): 3, r_t(4): 5}
+            d = self.newdict()
+            d[r_t(2)] = 3
+            d[r_t(4)] = 5

From noreply at buildbot.pypy.org  Tue Oct 29 10:53:51 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 29 Oct 2013 10:53:51 +0100 (CET)
Subject: [pypy-commit] stmgc default: fastpaths for
	stm_pointer_equal(_prebuilt)
Message-ID: <20131029095351.2C3FD1C1050@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r541:73c77375a8a6
Date: 2013-10-29 10:53 +0100
http://bitbucket.org/pypy/stmgc/changeset/73c77375a8a6/

Log:	fastpaths for stm_pointer_equal(_prebuilt)

diff --git a/c4/et.h b/c4/et.h
--- a/c4/et.h
+++ b/c4/et.h
@@ -72,7 +72,7 @@
 static const revision_t GCFLAG_OLD                    = STM_FIRST_GCFLAG << 0;
 static const revision_t GCFLAG_VISITED                = STM_FIRST_GCFLAG << 1;
 static const revision_t GCFLAG_PUBLIC                 = STM_FIRST_GCFLAG << 2;
-static const revision_t GCFLAG_PREBUILT_ORIGINAL      = STM_FIRST_GCFLAG << 3;
+// in stmgc.h:          GCFLAG_PREBUILT_ORIGINAL      = STM_FIRST_GCFLAG << 3;
 // in stmgc.h:          GCFLAG_PUBLIC_TO_PRIVATE      = STM_FIRST_GCFLAG << 4;
 // in stmgc.h:          GCFLAG_WRITE_BARRIER          = STM_FIRST_GCFLAG << 5;
 // in stmgc.h:          GCFLAG_MOVED                  = STM_FIRST_GCFLAG << 6;
diff --git a/c4/extra.c b/c4/extra.c
--- a/c4/extra.c
+++ b/c4/extra.c
@@ -246,9 +246,10 @@
     return result;
 }
 
-_Bool stm_pointer_equal(gcptr p1, gcptr p2)
+_Bool stm_direct_pointer_equal(gcptr p1, gcptr p2)
 {
-    if (p1 != NULL && p2 != NULL) {
+    /* commented lines are in the fastpath in stmgc.h */
+    /* if (p1 != NULL && p2 != NULL) { */
         /* resolve h_original, but only if !PREBUILT_ORIGINAL */
         if (p1->h_original && !(p1->h_tid & GCFLAG_PREBUILT_ORIGINAL)) {
             p1 = (gcptr)p1->h_original;
@@ -256,22 +257,23 @@
         if (p2->h_original && !(p2->h_tid & GCFLAG_PREBUILT_ORIGINAL)) {
             p2 = (gcptr)p2->h_original;
         }
-    }
+    /* } */
     return (p1 == p2);
 }
 
-_Bool stm_pointer_equal_prebuilt(gcptr p1, gcptr p2)
-{
-    assert(p2 != NULL);
-    assert(p2->h_tid & GCFLAG_PREBUILT_ORIGINAL);
+/* FULLY IMPLEMENTED AS MACRO IN stmgc.h */
+/* _Bool stm_pointer_equal_prebuilt(gcptr p1, gcptr p2) */
+/* { */
+/*     assert(p2 != NULL); */
+/*     assert(p2->h_tid & GCFLAG_PREBUILT_ORIGINAL); */
 
-    if (p1 == p2)
-        return 1;
+/*     if (p1 == p2) */
+/*         return 1; */
 
-    /* the only possible case to still get True is if p2 == p1->h_original */
-    return (p1 != NULL) && (p1->h_original == (revision_t)p2) &&
-        !(p1->h_tid & GCFLAG_PREBUILT_ORIGINAL);
-}
+/*     /\* the only possible case to still get True is if p2 == p1->h_original *\/ */
+/*     return (p1 != NULL) && (p1->h_original == (revision_t)p2) && */
+/*         !(p1->h_tid & GCFLAG_PREBUILT_ORIGINAL); */
+/* } */
 
 /************************************************************/
 
diff --git a/c4/nursery.c b/c4/nursery.c
--- a/c4/nursery.c
+++ b/c4/nursery.c
@@ -94,7 +94,7 @@
     return P;
 }
 
-gcptr stm_allocate(size_t size, unsigned long tid)
+inline gcptr stm_allocate(size_t size, unsigned long tid)
 {
     /* XXX inline the fast path */
     assert(tid == (tid & STM_USER_TID_MASK));
diff --git a/c4/stmgc.h b/c4/stmgc.h
--- a/c4/stmgc.h
+++ b/c4/stmgc.h
@@ -31,7 +31,7 @@
 /* push roots around allocating functions! */
 
 /* allocate an object out of the local nursery */
-gcptr stm_allocate(size_t size, unsigned long tid);
+inline gcptr stm_allocate(size_t size, unsigned long tid);
 /* allocate an object that is be immutable. it cannot be changed with
    a stm_write_barrier() or after the next commit */
 gcptr stm_allocate_immutable(size_t size, unsigned long tid);
@@ -49,8 +49,10 @@
 revision_t stm_id(gcptr);
 /* returns nonzero if the two object-copy pointers belong to the
 same original object */
+#if 0 // (optimized version below)
 _Bool stm_pointer_equal(gcptr, gcptr);
 _Bool stm_pointer_equal_prebuilt(gcptr, gcptr); /* 2nd arg is known prebuilt */
+#endif
 
 /* to push/pop objects into the local shadowstack */
 #if 0     // (optimized version below)
@@ -209,6 +211,7 @@
 gcptr stm_RepeatReadBarrier(gcptr);
 gcptr stm_ImmutReadBarrier(gcptr);
 gcptr stm_RepeatWriteBarrier(gcptr);
+static const revision_t GCFLAG_PREBUILT_ORIGINAL = STM_FIRST_GCFLAG << 3;
 static const revision_t GCFLAG_PUBLIC_TO_PRIVATE = STM_FIRST_GCFLAG << 4;
 static const revision_t GCFLAG_WRITE_BARRIER = STM_FIRST_GCFLAG << 5;
 static const revision_t GCFLAG_MOVED = STM_FIRST_GCFLAG << 6;
@@ -220,6 +223,16 @@
 
 #define UNLIKELY(test)  __builtin_expect(test, 0)
 
+_Bool stm_direct_pointer_equal(gcptr, gcptr);
+#define stm_pointer_equal(p1, p2)                    \
+    (((p1) == (p2))                                  \
+    || ((p1) != NULL && (p2) != NULL                 \
+        && stm_direct_pointer_equal(p1, p2)))
+#define stm_pointer_equal_prebuilt(p1, p2)           \
+    (((p1) == (p2))                                  \
+     || (((p1) != NULL) && ((p1)->h_original == (revision_t)(p2)) &&    \
+         !((p1)->h_tid & GCFLAG_PREBUILT_ORIGINAL)))
+
 #ifdef STM_BARRIER_COUNT
 # define STM_BARRIER_NUMBERS  12
 # define STM_BARRIER_NAMES "stm_read_barrier\n"         \

From noreply at buildbot.pypy.org  Tue Oct 29 10:55:15 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 29 Oct 2013 10:55:15 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c4: improve code generation for
 transaction_break and pointer_equal
Message-ID: <20131029095515.355FA1C1050@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67679:f6fa761a828e
Date: 2013-10-28 14:58 +0100
http://bitbucket.org/pypy/pypy/changeset/f6fa761a828e/

Log:	improve code generation for transaction_break and pointer_equal

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,3 +1,8 @@
+------------------------------------------------------------
+
+make stm_transaction_break use cond_call (or other ways to not
+spill all registers)
+
 ------------------------------------------------------------
 
 constptrs always require slowpath of read_barrier if they
diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -319,8 +319,8 @@
         lst = op.getarglist()
         lst[1] = self.gen_barrier(lst[1], 'W')
         op = op.copy_and_change(op.getopnum(), args=lst)
-        # then a read barrier the source string
-        self.handle_category_operations(op, 'R')
+        # then an immutable read barrier the source string
+        self.handle_category_operations(op, 'I')
 
     @specialize.arg(1)
     def _do_stm_call(self, funcname, args, result):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -425,16 +425,12 @@
 
     def _build_ptr_eq_slowpath(self):
         cpu = self.cpu
-        is_stm = cpu.gc_ll_descr.stm
-        assert is_stm
-
-        func = cpu.gc_ll_descr.get_malloc_fn_addr('stm_ptr_eq')
+        assert cpu.gc_ll_descr.stm
         #
         # This builds a helper function called from the slow path of
         # ptr_eq/ne.  It must save all registers, and optionally
-        # all XMM registers.  It takes a single argument just pushed
-        # on the stack even on X86_64.  It must restore stack alignment
-        # accordingly.
+        # all XMM registers. It takes two values pushed on the stack,
+        # even on X86_64.  It must restore stack alignment accordingly.
         mc = codebuf.MachineCodeBlockWrapper()
         #
         self._push_all_regs_to_frame(mc, [], withfloats=False,
@@ -457,7 +453,12 @@
             mc.MOV_rs(edi.value, 2 * WORD)
             mc.MOV_rs(esi.value, 3 * WORD)
         #
-        mc.CALL(imm(func))
+        if not we_are_translated(): # for tests
+            fn = cpu.gc_ll_descr.get_malloc_fn_addr('stm_ptr_eq')
+            mc.CALL(imm(fn))
+        else:
+            fn = stmtlocal.stm_pointer_equal_fn
+            mc.CALL(imm(self.cpu.cast_ptr_to_int(fn)))
         # eax has result
         if IS_X86_32:
             # ||val2|val1|retaddr|x||x|x|val2|val1|
@@ -2367,12 +2368,16 @@
         self.mc.overwrite(jmp_location - 1, chr(offset))
 
     # ------------------- END CALL ASSEMBLER -----------------------
-    def _stm_ptr_eq_fastpath(self, mc, arglocs, result_loc):
+    def _stm_ptr_eq_fastpath(self, mc, arglocs):
         assert self.cpu.gc_ll_descr.stm
         assert self.ptr_eq_slowpath is not None
         a_base = arglocs[0]
         b_base = arglocs[1]
-
+        if isinstance(a_base, ImmedLoc):
+            # make sure there is a non-immed as the first
+            # argument to mc.CMP(). (2 immeds are caught below)
+            a_base, b_base = b_base, a_base
+        
         #
         # FASTPATH
         #
@@ -2386,25 +2391,11 @@
             else:
                 j_ok1 = 0
         else:
-            # do the dance, even if a or b is an Immed
-            # XXX: figure out if CMP() is able to handle it without
-            #      the explicit MOV before it (CMP(a_base, b_base))
+            mc.CMP(a_base, b_base)
+            # reverse flags: if p1==p2, set NZ
             sl = X86_64_SCRATCH_REG.lowest8bits()
-            mc.MOV(X86_64_SCRATCH_REG, a_base)
-            if isinstance(b_base, ImmedLoc) \
-              and rx86.fits_in_32bits(b_base.value):
-                mc.CMP_ri(X86_64_SCRATCH_REG.value, b_base.value)
-            elif not isinstance(b_base, ImmedLoc):
-                mc.CMP(X86_64_SCRATCH_REG, b_base)
-            else:
-                # imm64, need another temporary reg :(
-                mc.PUSH_r(eax.value)
-                mc.MOV_ri64(eax.value, b_base.value)
-                mc.CMP_rr(X86_64_SCRATCH_REG.value, eax.value)
-                mc.POP_r(eax.value)
-            # reverse flags: if p1==p2, set NZ
             mc.SET_ir(rx86.Conditions['Z'], sl.value)
-            mc.AND8_rr(sl.value, sl.value)
+            mc.TEST8_rr(sl.value, sl.value)
             mc.J_il8(rx86.Conditions['NZ'], 0)
             j_ok1 = mc.get_relative_pos()
 
@@ -3163,8 +3154,37 @@
         # call stm_transaction_break() with the address of the
         # STM_RESUME_BUF and the custom longjmp function
         self.push_gcmap(mc, gcmap, mov=True)
+        #
+        # save all registers
+        base_ofs = self.cpu.get_baseofs_of_frame_field()
+        for gpr in self._regalloc.rm.reg_bindings.values():
+            v = gpr_reg_mgr_cls.all_reg_indexes[gpr.value]
+            mc.MOV_br(v * WORD + base_ofs, gpr.value)
+        if IS_X86_64:
+            coeff = 1
+        else:
+            coeff = 2
+        ofs = len(gpr_reg_mgr_cls.all_regs)
+        for xr in self._regalloc.xrm.reg_bindings.values():
+            mc.MOVSD_bx((ofs + xr.value * coeff) * WORD + base_ofs, xr.value)
+        #
+        # CALL break function
         fn = self.stm_transaction_break_path
         mc.CALL(imm(fn))
+        # HERE is the place an aborted transaction retries
+        #
+        # restore regs
+        base_ofs = self.cpu.get_baseofs_of_frame_field()
+        for gpr in self._regalloc.rm.reg_bindings.values():
+            v = gpr_reg_mgr_cls.all_reg_indexes[gpr.value]
+            mc.MOV_rb(gpr.value, v * WORD + base_ofs)
+        if IS_X86_64:
+            coeff = 1
+        else:
+            coeff = 2
+        ofs = len(gpr_reg_mgr_cls.all_regs)
+        for xr in self._regalloc.xrm.reg_bindings.values():
+            mc.MOVSD_xb(xr.value, (ofs + xr.value * coeff) * WORD + base_ofs)
         #
         # patch the JZ above
         if jz_location:
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -412,11 +412,9 @@
         args = op.getarglist()
         if args[0].type == REF:
             assert args[1].type == REF
-            # XXX: this is certainly not wanted.
-            # We force immed64 into registers here.
-            x = self.make_sure_var_in_reg(args[0], args, selected_reg=ecx)
-            y = self.make_sure_var_in_reg(args[1], args, selected_reg=eax)
-            self.rm.possibly_free_var(args[1])
+            # move both args to reg or immed
+            x = self.make_sure_var_in_reg(args[0], args)
+            y = self.make_sure_var_in_reg(args[1], args)
         else:
             x = self.make_sure_var_in_reg(args[0], args)
             y = self.loc(args[1])
@@ -1288,10 +1286,9 @@
         assert isinstance(check_type_box, ConstInt)
         check_type = check_type_box.getint()
         #
-        # XXX use the extra 3 words in the stm resume buffer to save
-        # up to 3 registers, too.  For now we just flush them all.
-        self.xrm.before_call(save_all_regs=1)
-        self.rm.before_call(save_all_regs=1)
+        # only save regs for the should_break_transaction call
+        self.xrm.before_call()
+        self.rm.before_call()
         gcmap = self.get_gcmap() # allocate the gcmap *before*
         #
         self.assembler.stm_transaction_break(check_type, gcmap)
diff --git a/rpython/jit/backend/x86/stmtlocal.py b/rpython/jit/backend/x86/stmtlocal.py
--- a/rpython/jit/backend/x86/stmtlocal.py
+++ b/rpython/jit/backend/x86/stmtlocal.py
@@ -47,3 +47,9 @@
     'stm_invalidate_jmp_buf',
     [llmemory.Address], lltype.Void,
     sandboxsafe=True, _nowrapper=True, transactionsafe=True)
+stm_pointer_equal_fn = rffi.llexternal(
+    'stm_pointer_equal',
+    [llmemory.Address, llmemory.Address], lltype.Bool,
+    sandboxsafe=True, _nowrapper=True, transactionsafe=True)
+
+
diff --git a/rpython/jit/backend/x86/test/test_stm_integration.py b/rpython/jit/backend/x86/test/test_stm_integration.py
--- a/rpython/jit/backend/x86/test/test_stm_integration.py
+++ b/rpython/jit/backend/x86/test/test_stm_integration.py
@@ -835,8 +835,8 @@
                     args = [s for i, s in enumerate((s1, s2))
                             if not isinstance((p1, p2)[i], Const)] + [7]
                                         
-                    frame = self.cpu.execute_token(looptoken, *args)
-                    frame = rffi.cast(JITFRAMEPTR, frame)
+                    deadframe = self.cpu.execute_token(looptoken, *args)
+                    frame = rffi.cast(JITFRAMEPTR, deadframe)
                     frame_adr = rffi.cast(lltype.Signed, frame.jf_descr)
                     guard_failed = frame_adr != id(finaldescr)
 
@@ -849,8 +849,10 @@
                         
                     if a == b or a == 0 or b == 0:
                         assert (a, b) not in called_on
+                        assert (b, a) not in called_on
                     else:
-                        assert [(a, b)] == called_on
+                        assert ([(a, b)] == called_on
+                                or [(b, a)] == called_on)
 
                     if guard is not None:
                         if a == b:

From noreply at buildbot.pypy.org  Tue Oct 29 10:55:16 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 29 Oct 2013 10:55:16 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c4: stupid
Message-ID: <20131029095516.5C4671C1050@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67680:f1af6552087b
Date: 2013-10-28 15:03 +0100
http://bitbucket.org/pypy/pypy/changeset/f1af6552087b/

Log:	stupid

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2368,7 +2368,7 @@
         self.mc.overwrite(jmp_location - 1, chr(offset))
 
     # ------------------- END CALL ASSEMBLER -----------------------
-    def _stm_ptr_eq_fastpath(self, mc, arglocs):
+    def _stm_ptr_eq_fastpath(self, mc, arglocs, result_loc):
         assert self.cpu.gc_ll_descr.stm
         assert self.ptr_eq_slowpath is not None
         a_base = arglocs[0]

From noreply at buildbot.pypy.org  Tue Oct 29 10:55:17 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 29 Oct 2013 10:55:17 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c4: don't remove debug_merge_points in
 stmrewrite (for jitlog)
Message-ID: <20131029095517.6C7AA1C1050@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67681:9c9899db05fe
Date: 2013-10-28 20:22 +0100
http://bitbucket.org/pypy/pypy/changeset/9c9899db05fe/

Log:	don't remove debug_merge_points in stmrewrite (for jitlog)

diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -46,11 +46,9 @@
             if not we_are_translated():
                 # only possible in tests:
                 if opnum in (rop.COND_CALL_STM_B, 
-                                     -124): # FORCE_SPILL
+                            -124): # FORCE_SPILL
                     self.newops.append(op)
                     continue
-            if opnum == rop.DEBUG_MERGE_POINT:
-                continue
             if opnum == rop.INCREMENT_DEBUG_COUNTER:
                 self.newops.append(op)
                 continue

From noreply at buildbot.pypy.org  Tue Oct 29 10:55:18 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 29 Oct 2013 10:55:18 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c4: jitlogparser output improvement
Message-ID: <20131029095518.9697D1C1050@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67682:b79fa946dff3
Date: 2013-10-28 20:23 +0100
http://bitbucket.org/pypy/pypy/changeset/b79fa946dff3/

Log:	jitlogparser output improvement

diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -87,7 +87,7 @@
                     continue
                 e = elem.split("\t")
                 adr = e[0]
-                v = elem   # --- more compactly:  " ".join(e[2:])
+                v = " ".join(e[2:])
                 if not start:
                     start = int(adr.strip(":"), 16)
                 ofs = int(adr.strip(":"), 16)
@@ -102,7 +102,6 @@
                 if ofs >= 0:
                     asm.append((ofs, v.strip("\n")))
             #
-            prefix = hex(dump_start)[:-9]
             asm_index = 0
             for i, op in enumerate(loop.operations):
                 end = 0
@@ -122,7 +121,7 @@
                     while asm[end_index][0] - start < end and end_index < len(asm) - 1:
                         end_index += 1
                     op.asm = '\n'.join([
-                        prefix+hex(asm[i][0])[2:] + ": " + asm[i][1] 
+                        hex(asm[i][0]) + ": " + asm[i][1] 
                         for i in range(asm_index, end_index)])
         return loop
 

From noreply at buildbot.pypy.org  Tue Oct 29 10:55:19 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 29 Oct 2013 10:55:19 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c4: implement stm_pointer_equal directly
 in the slowpath (less register saving overhead)
Message-ID: <20131029095519.AC98F1C1050@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67683:ede0da761764
Date: 2013-10-28 20:25 +0100
http://bitbucket.org/pypy/pypy/changeset/ede0da761764/

Log:	implement stm_pointer_equal directly in the slowpath (less register
	saving overhead)

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -427,52 +427,69 @@
         cpu = self.cpu
         assert cpu.gc_ll_descr.stm
         #
+        # SYNCHRONIZE WITH extra.c'S IMPLEMENTATION!
+        #
         # This builds a helper function called from the slow path of
         # ptr_eq/ne.  It must save all registers, and optionally
         # all XMM registers. It takes two values pushed on the stack,
         # even on X86_64.  It must restore stack alignment accordingly.
         mc = codebuf.MachineCodeBlockWrapper()
         #
-        self._push_all_regs_to_frame(mc, [], withfloats=False,
-                                     callee_only=True)
+        # we want 2 registers:
+        mc.PUSH_r(esi.value)
+        mc.PUSH_r(edi.value)
         #
-        if IS_X86_32:
-            # ||val2|val1|retaddr|  growing->, || aligned 
-            mc.SUB_ri(esp.value, 5 * WORD)
-            # ||val2|val1|retaddr|x||x|x|x|x|
-            mc.MOV_rs(eax.value, 6 * WORD)
-            mc.MOV_rs(ecx.value, 7 * WORD)
-            # eax=val1, ecx=val2
-            mc.MOV_sr(0, eax.value)
-            mc.MOV_sr(WORD, ecx.value)
-            # ||val2|val1|retaddr|x||x|x|val2|val1|
-        else:
-            # ||val2|val1||retaddr|
-            mc.SUB_ri(esp.value, WORD)
-            # ||val2|val1||retaddr|x||
-            mc.MOV_rs(edi.value, 2 * WORD)
-            mc.MOV_rs(esi.value, 3 * WORD)
+        # get arguments: ||val2|val1||retaddr|esi||edi|
+        mc.MOV_rs(esi.value, 3 * WORD)
+        mc.MOV_rs(edi.value, 4 * WORD)
         #
-        if not we_are_translated(): # for tests
-            fn = cpu.gc_ll_descr.get_malloc_fn_addr('stm_ptr_eq')
-            mc.CALL(imm(fn))
-        else:
-            fn = stmtlocal.stm_pointer_equal_fn
-            mc.CALL(imm(self.cpu.cast_ptr_to_int(fn)))
-        # eax has result
-        if IS_X86_32:
-            # ||val2|val1|retaddr|x||x|x|val2|val1|
-            mc.ADD_ri(esp.value, 5 * WORD)
-            # ||val2|val1|retaddr|
-        else:
-            # ||val2|val1||retaddr|x||
-            mc.ADD_ri(esp.value, WORD)
-            # ||val2|val1||retaddr|
-        mc.MOV_sr(2 * WORD, eax.value)
+        # the fastpath checks if val1==val2 or any of them is NULL
+        # thus, we only have to get to their h_original
+        # if they are *not* PREBUILT_ORIGINALS
+        #
+        flag = StmGC.GCFLAG_PREBUILT_ORIGINAL
+        assert (flag >> 32) > 0 and (flag >> 40) == 0
+        flag = flag >> 32
+        off = 4
+        # if !(val1->h_original), leave EDI as is
+        mc.MOV_rm(X86_64_SCRATCH_REG.value, (edi.value, StmGC.H_ORIGINAL))
+        mc.TEST_rr(X86_64_SCRATCH_REG.value, X86_64_SCRATCH_REG.value)
+        mc.J_il8(rx86.Conditions['Z'], 0)
+        z1_location = mc.get_relative_pos()
+        # if val1->h_tid & PREBUILT_ORIGINAL, take h_original
+        mc.TEST8_mi((edi.value, StmGC.H_TID + off), flag)
+        mc.CMOVE_rr(edi.value, X86_64_SCRATCH_REG.value)
+        #
+        # Do the same for val2=ESI
+        offset = mc.get_relative_pos() - z1_location
+        assert 0 < offset <= 127
+        mc.overwrite(z1_location - 1, chr(offset))
+        # if !(val2->h_original), leave ESI as is
+        mc.MOV_rm(X86_64_SCRATCH_REG.value, (esi.value, StmGC.H_ORIGINAL))
+        mc.TEST_rr(X86_64_SCRATCH_REG.value, X86_64_SCRATCH_REG.value)
+        mc.J_il8(rx86.Conditions['Z'], 0)
+        z2_location = mc.get_relative_pos()
+        # if val2->h_tid & PREBUILT_ORIGINAL, take h_original
+        mc.TEST8_mi((esi.value, StmGC.H_TID + off), flag)
+        mc.CMOVE_rr(esi.value, X86_64_SCRATCH_REG.value)
+        #
+        # COMPARE
+        offset = mc.get_relative_pos() - z2_location
+        assert 0 < offset <= 127
+        mc.overwrite(z2_location - 1, chr(offset))
+        #
+        mc.CMP_rr(edi.value, esi.value)
+        sl = X86_64_SCRATCH_REG.lowest8bits()
+        mc.SET_ir(rx86.Conditions['Z'], sl.value)
+        # mov result to val2 on stack
+        # ||val2|val1||retaddr|esi||edi|
+        mc.MOV_sr(4 * WORD, X86_64_SCRATCH_REG.value)
+        #
+        # Restore everything:
+        mc.POP_r(edi.value)
+        mc.POP_r(esi.value)
         # ||result|val1|retaddr|
         #
-        self._pop_all_regs_from_frame(mc, [], withfloats=False,
-                                      callee_only=True)
         #
         # only remove one arg:
         mc.RET16_i(1 * WORD)
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -530,6 +530,7 @@
     NOT_b = insn(rex_w, '\xF7', orbyte(2<<3), stack_bp(1))
 
     CMOVNS_rr = insn(rex_w, '\x0F\x49', register(1, 8), register(2), '\xC0')
+    CMOVE_rr = insn(rex_w, '\x0F\x44', register(1, 8), register(2), '\xC0')
 
     # ------------------------------ Misc stuff ------------------------------
 
diff --git a/rpython/jit/backend/x86/test/test_stm_integration.py b/rpython/jit/backend/x86/test/test_stm_integration.py
--- a/rpython/jit/backend/x86/test/test_stm_integration.py
+++ b/rpython/jit/backend/x86/test/test_stm_integration.py
@@ -847,12 +847,16 @@
                     if isinstance(p2, Const):
                         b = cast_to_int(p2.value)
                         
-                    if a == b or a == 0 or b == 0:
-                        assert (a, b) not in called_on
-                        assert (b, a) not in called_on
-                    else:
-                        assert ([(a, b)] == called_on
-                                or [(b, a)] == called_on)
+                    # XXX: there is now no function being called in the
+                    # slowpath, so we can't check if fast- vs. slowpath
+                    # works :/
+                    
+                    # if a == b or a == 0 or b == 0:
+                    #     assert (a, b) not in called_on
+                    #     assert (b, a) not in called_on
+                    # else:
+                    #     assert ([(a, b)] == called_on
+                    #             or [(b, a)] == called_on)
 
                     if guard is not None:
                         if a == b:

From noreply at buildbot.pypy.org  Tue Oct 29 10:55:20 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 29 Oct 2013 10:55:20 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c4: fix previous fix for debug_merge_point
 (always turned inevitable)
Message-ID: <20131029095520.C9FE71C1050@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67684:c42ba7a70b35
Date: 2013-10-29 09:31 +0100
http://bitbucket.org/pypy/pypy/changeset/c42ba7a70b35/

Log:	fix previous fix for debug_merge_point (always turned inevitable)

diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py
--- a/rpython/jit/backend/llsupport/stmrewrite.py
+++ b/rpython/jit/backend/llsupport/stmrewrite.py
@@ -49,13 +49,14 @@
                             -124): # FORCE_SPILL
                     self.newops.append(op)
                     continue
-            if opnum == rop.INCREMENT_DEBUG_COUNTER:
+            if opnum in (rop.INCREMENT_DEBUG_COUNTER,
+                         rop.DEBUG_MERGE_POINT):
                 self.newops.append(op)
                 continue
             # ----------  ptr_eq  ----------
             if opnum in (rop.PTR_EQ, rop.INSTANCE_PTR_EQ,
                          rop.PTR_NE, rop.INSTANCE_PTR_NE):
-                self.handle_ptr_eq(op)
+                self.newops.append(op)
                 continue
             # ----------  guard_class  ----------
             if opnum == rop.GUARD_CLASS:
@@ -338,9 +339,6 @@
     def _is_null(self, box):
         return isinstance(box, ConstPtr) and not box.value
 
-    def handle_ptr_eq(self, op):
-        self.newops.append(op)
-
     def maybe_handle_raw_accesses(self, op):
         from rpython.jit.backend.llsupport.descr import FieldDescr
         descr = op.getdescr()
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -410,14 +410,8 @@
 
     def consider_guard_value(self, op):
         args = op.getarglist()
-        if args[0].type == REF:
-            assert args[1].type == REF
-            # move both args to reg or immed
-            x = self.make_sure_var_in_reg(args[0], args)
-            y = self.make_sure_var_in_reg(args[1], args)
-        else:
-            x = self.make_sure_var_in_reg(args[0], args)
-            y = self.loc(args[1])
+        x = self.make_sure_var_in_reg(args[0], args)
+        y = self.loc(args[1])
         self.perform_guard(op, [x, y], None)
 
     def consider_guard_class(self, op):

From noreply at buildbot.pypy.org  Tue Oct 29 10:55:21 2013
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 29 Oct 2013 10:55:21 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c4: import stmgc (fastpaths for
 stm_pointer_equal(_prebuilt))
Message-ID: <20131029095521.EDB621C1050@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c4
Changeset: r67685:c3a3d6864203
Date: 2013-10-29 10:54 +0100
http://bitbucket.org/pypy/pypy/changeset/c3a3d6864203/

Log:	import stmgc (fastpaths for stm_pointer_equal(_prebuilt))

diff --git a/rpython/translator/stm/src_stm/et.h b/rpython/translator/stm/src_stm/et.h
--- a/rpython/translator/stm/src_stm/et.h
+++ b/rpython/translator/stm/src_stm/et.h
@@ -73,7 +73,7 @@
 static const revision_t GCFLAG_OLD                    = STM_FIRST_GCFLAG << 0;
 static const revision_t GCFLAG_VISITED                = STM_FIRST_GCFLAG << 1;
 static const revision_t GCFLAG_PUBLIC                 = STM_FIRST_GCFLAG << 2;
-static const revision_t GCFLAG_PREBUILT_ORIGINAL      = STM_FIRST_GCFLAG << 3;
+// in stmgc.h:          GCFLAG_PREBUILT_ORIGINAL      = STM_FIRST_GCFLAG << 3;
 // in stmgc.h:          GCFLAG_PUBLIC_TO_PRIVATE      = STM_FIRST_GCFLAG << 4;
 // in stmgc.h:          GCFLAG_WRITE_BARRIER          = STM_FIRST_GCFLAG << 5;
 // in stmgc.h:          GCFLAG_MOVED                  = STM_FIRST_GCFLAG << 6;
diff --git a/rpython/translator/stm/src_stm/extra.c b/rpython/translator/stm/src_stm/extra.c
--- a/rpython/translator/stm/src_stm/extra.c
+++ b/rpython/translator/stm/src_stm/extra.c
@@ -247,9 +247,10 @@
     return result;
 }
 
-_Bool stm_pointer_equal(gcptr p1, gcptr p2)
+_Bool stm_direct_pointer_equal(gcptr p1, gcptr p2)
 {
-    if (p1 != NULL && p2 != NULL) {
+    /* commented lines are in the fastpath in stmgc.h */
+    /* if (p1 != NULL && p2 != NULL) { */
         /* resolve h_original, but only if !PREBUILT_ORIGINAL */
         if (p1->h_original && !(p1->h_tid & GCFLAG_PREBUILT_ORIGINAL)) {
             p1 = (gcptr)p1->h_original;
@@ -257,22 +258,23 @@
         if (p2->h_original && !(p2->h_tid & GCFLAG_PREBUILT_ORIGINAL)) {
             p2 = (gcptr)p2->h_original;
         }
-    }
+    /* } */
     return (p1 == p2);
 }
 
-_Bool stm_pointer_equal_prebuilt(gcptr p1, gcptr p2)
-{
-    assert(p2 != NULL);
-    assert(p2->h_tid & GCFLAG_PREBUILT_ORIGINAL);
+/* FULLY IMPLEMENTED AS MACRO IN stmgc.h */
+/* _Bool stm_pointer_equal_prebuilt(gcptr p1, gcptr p2) */
+/* { */
+/*     assert(p2 != NULL); */
+/*     assert(p2->h_tid & GCFLAG_PREBUILT_ORIGINAL); */
 
-    if (p1 == p2)
-        return 1;
+/*     if (p1 == p2) */
+/*         return 1; */
 
-    /* the only possible case to still get True is if p2 == p1->h_original */
-    return (p1 != NULL) && (p1->h_original == (revision_t)p2) &&
-        !(p1->h_tid & GCFLAG_PREBUILT_ORIGINAL);
-}
+/*     /\* the only possible case to still get True is if p2 == p1->h_original *\/ */
+/*     return (p1 != NULL) && (p1->h_original == (revision_t)p2) && */
+/*         !(p1->h_tid & GCFLAG_PREBUILT_ORIGINAL); */
+/* } */
 
 /************************************************************/
 
diff --git a/rpython/translator/stm/src_stm/nursery.c b/rpython/translator/stm/src_stm/nursery.c
--- a/rpython/translator/stm/src_stm/nursery.c
+++ b/rpython/translator/stm/src_stm/nursery.c
@@ -95,7 +95,7 @@
     return P;
 }
 
-gcptr stm_allocate(size_t size, unsigned long tid)
+inline gcptr stm_allocate(size_t size, unsigned long tid)
 {
     /* XXX inline the fast path */
     assert(tid == (tid & STM_USER_TID_MASK));
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-ba0819e4b5e7
+73c77375a8a6
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -32,7 +32,7 @@
 /* push roots around allocating functions! */
 
 /* allocate an object out of the local nursery */
-gcptr stm_allocate(size_t size, unsigned long tid);
+inline gcptr stm_allocate(size_t size, unsigned long tid);
 /* allocate an object that is be immutable. it cannot be changed with
    a stm_write_barrier() or after the next commit */
 gcptr stm_allocate_immutable(size_t size, unsigned long tid);
@@ -50,8 +50,10 @@
 revision_t stm_id(gcptr);
 /* returns nonzero if the two object-copy pointers belong to the
 same original object */
+#if 0 // (optimized version below)
 _Bool stm_pointer_equal(gcptr, gcptr);
 _Bool stm_pointer_equal_prebuilt(gcptr, gcptr); /* 2nd arg is known prebuilt */
+#endif
 
 /* to push/pop objects into the local shadowstack */
 #if 0     // (optimized version below)
@@ -210,6 +212,7 @@
 gcptr stm_RepeatReadBarrier(gcptr);
 gcptr stm_ImmutReadBarrier(gcptr);
 gcptr stm_RepeatWriteBarrier(gcptr);
+static const revision_t GCFLAG_PREBUILT_ORIGINAL = STM_FIRST_GCFLAG << 3;
 static const revision_t GCFLAG_PUBLIC_TO_PRIVATE = STM_FIRST_GCFLAG << 4;
 static const revision_t GCFLAG_WRITE_BARRIER = STM_FIRST_GCFLAG << 5;
 static const revision_t GCFLAG_MOVED = STM_FIRST_GCFLAG << 6;
@@ -221,6 +224,16 @@
 
 #define UNLIKELY(test)  __builtin_expect(test, 0)
 
+_Bool stm_direct_pointer_equal(gcptr, gcptr);
+#define stm_pointer_equal(p1, p2)                    \
+    (((p1) == (p2))                                  \
+    || ((p1) != NULL && (p2) != NULL                 \
+        && stm_direct_pointer_equal(p1, p2)))
+#define stm_pointer_equal_prebuilt(p1, p2)           \
+    (((p1) == (p2))                                  \
+     || (((p1) != NULL) && ((p1)->h_original == (revision_t)(p2)) &&    \
+         !((p1)->h_tid & GCFLAG_PREBUILT_ORIGINAL)))
+
 #ifdef STM_BARRIER_COUNT
 # define STM_BARRIER_NUMBERS  12
 # define STM_BARRIER_NAMES "stm_read_barrier\n"         \

From noreply at buildbot.pypy.org  Tue Oct 29 20:32:55 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:32:55 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: start branch to work towards
	removing numpypy
Message-ID: <20131029193255.228E11C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67686:8e4109fc1c34
Date: 2013-10-29 05:57 -0400
http://bitbucket.org/pypy/pypy/changeset/8e4109fc1c34/

Log:	start branch to work towards removing numpypy


From noreply at buildbot.pypy.org  Tue Oct 29 20:32:56 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:32:56 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: eliminate reliance on numpypy in
 most of test_numarray
Message-ID: <20131029193256.AE12E1C0163@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67687:636e0e00d1be
Date: 2013-10-29 05:58 -0400
http://bitbucket.org/pypy/pypy/changeset/636e0e00d1be/

Log:	eliminate reliance on numpypy in most of test_numarray

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -869,7 +869,8 @@
             assert b[i] == i - 5
 
     def test_scalar_subtract(self):
-        from numpypy import int32
+        from numpypy import dtype
+        int32 = dtype('int32').type
         assert int32(2) - 1 == 1
         assert 1 - int32(2) == -1
 
@@ -885,9 +886,10 @@
         a = numpypy.array(range(5), dtype=bool)
         b = a * a
         assert b.dtype is numpypy.dtype(bool)
-        assert b[0] is numpypy.False_
+        bool_ = numpypy.dtype(bool).type
+        assert b[0] is bool_(False)
         for i in range(1, 5):
-            assert b[i] is numpypy.True_
+            assert b[i] is bool_(True)
 
     def test_mul_constant(self):
         from numpypy import array
@@ -1254,7 +1256,7 @@
         assert list(zeros((0, 2)).sum(axis=1)) == []
 
     def test_reduce_nd(self):
-        from numpypy import arange, array, multiply
+        from numpypy import arange, array
         a = arange(15).reshape(5, 3)
         assert a.sum() == 105
         assert a.max() == 14
@@ -1272,7 +1274,6 @@
         assert ((a + a).max() == 28)
         assert ((a + a).max(0) == [24, 26, 28]).all()
         assert ((a + a).sum(1) == [6, 24, 42, 60, 78]).all()
-        assert (multiply.reduce(a) == array([0, 3640, 12320])).all()
         a = array(range(105)).reshape(3, 5, 7)
         assert (a[:, 1, :].sum(0) == [126, 129, 132, 135, 138, 141, 144]).all()
         assert (a[:, 1, :].sum(1) == [70, 315, 560]).all()
@@ -1365,7 +1366,7 @@
         assert c.any() == False
 
     def test_dtype_guessing(self):
-        from numpypy import array, dtype, float64, int8, bool_
+        from numpypy import array, dtype
 
         assert array([True]).dtype is dtype(bool)
         assert array([True, False]).dtype is dtype(bool)
@@ -1375,6 +1376,9 @@
         assert array([1.2, True]).dtype is dtype(float)
         assert array([1.2, 5]).dtype is dtype(float)
         assert array([]).dtype is dtype(float)
+        float64 = dtype('float64').type
+        int8 = dtype('int8').type
+        bool_ = dtype('bool').type
         assert array([float64(2)]).dtype is dtype(float)
         assert array([int8(3)]).dtype is dtype("int8")
         assert array([bool_(True)]).dtype is dtype(bool)
@@ -1479,19 +1483,11 @@
         assert arange(4, dtype='>c8').real.max() == 3.0
         assert arange(4, dtype='<c8').real.max() == 3.0
 
-    def test_trace(self):
-        import numpypy as np
-        assert np.trace(np.eye(3)) == 3.0
-        a = np.arange(8).reshape((2,2,2))
-        assert np.array_equal(np.trace(a), [6, 8])
-        a = np.arange(24).reshape((2,2,2,3))
-        assert np.trace(a).shape == (2, 3)
-
     def test_view(self):
-        from numpypy import array, int8, int16, dtype
-        x = array((1, 2), dtype=int8)
+        from numpypy import array, dtype
+        x = array((1, 2), dtype='int8')
         assert x.shape == (2,)
-        y = x.view(dtype=int16)
+        y = x.view(dtype='int16')
         assert x.shape == (2,)
         assert y[0] == 513
         assert y.dtype == dtype('int16')
@@ -1510,24 +1506,26 @@
         assert x.view('int8').shape == (10, 3)
 
     def test_ndarray_view_empty(self):
-        from numpypy import array, int8, int16
-        x = array([], dtype=[('a', int8), ('b', int8)])
-        y = x.view(dtype=int16)
+        from numpypy import array, dtype
+        x = array([], dtype=[('a', 'int8'), ('b', 'int8')])
+        y = x.view(dtype='int16')
 
     def test_scalar_view(self):
-        from numpypy import int64, array
+        from numpypy import dtype, array
         a = array(0, dtype='int32')
         b = a.view(dtype='float32')
         assert b.shape == ()
         assert b == 0
-        s = int64(12)
+        s = dtype('int64').type(12)
         exc = raises(ValueError, s.view, 'int8')
         assert exc.value[0] == "new type not compatible with array."
         skip('not implemented yet')
         assert s.view('double') < 7e-323
 
     def test_tolist_scalar(self):
-        from numpypy import int32, bool_
+        from numpypy import dtype
+        int32 = dtype('int32').type
+        bool_ = dtype('bool').type
         x = int32(23)
         assert x.tolist() == 23
         assert type(x.tolist()) is int
@@ -1705,12 +1703,13 @@
         assert array(3.0).nbytes == 8
 
     def test_repeat(self):
-        from numpypy import repeat, array
-        assert (repeat([[1, 2], [3, 4]], 3) == [1, 1, 1, 2, 2, 2,
-                                                3, 3, 3, 4, 4, 4]).all()
-        assert (repeat([[1, 2], [3, 4]], 2, axis=0) == [[1, 2], [1, 2], [3, 4],
-                                                        [3, 4]]).all()
-        assert (repeat([[1, 2], [3, 4]], 2, axis=1) == [[1, 1, 2, 2], [3, 3,
+        from numpypy import array
+        a = array([[1, 2], [3, 4]])
+        assert (a.repeat(3) == [1, 1, 1, 2, 2, 2,
+                                 3, 3, 3, 4, 4, 4]).all()
+        assert (a.repeat(2, axis=0) == [[1, 2], [1, 2], [3, 4],
+                                         [3, 4]]).all()
+        assert (a.repeat(2, axis=1) == [[1, 1, 2, 2], [3, 3,
                                                         4, 4]]).all()
         assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
@@ -2069,12 +2068,6 @@
         a = array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]])
         assert (a + a)[1, 1] == 8
 
-    def test_ufunc_negative(self):
-        from numpypy import array, negative
-        a = array([[1, 2], [3, 4]])
-        b = negative(a + a)
-        assert (b == [[-2, -4], [-6, -8]]).all()
-
     def test_getitem_3(self):
         from numpypy import array
         a = array([[1, 2], [3, 4], [5, 6], [7, 8],
@@ -2600,50 +2593,50 @@
 
     def test_fromstring(self):
         import sys
-        from numpypy import fromstring, uint8, float32, int32
+        from numpypy import fromstring, dtype
 
         a = fromstring(self.data)
         for i in range(4):
             assert a[i] == i + 1
-        b = fromstring('\x01\x02', dtype=uint8)
+        b = fromstring('\x01\x02', dtype='uint8')
         assert a[0] == 1
         assert a[1] == 2
-        c = fromstring(self.fdata, dtype=float32)
-        assert c[0] == float32(2.3)
-        d = fromstring("1 2", sep=' ', count=2, dtype=uint8)
+        c = fromstring(self.fdata, dtype='float32')
+        assert c[0] == dtype('float32').type(2.3)
+        d = fromstring("1 2", sep=' ', count=2, dtype='uint8')
         assert len(d) == 2
         assert d[0] == 1
         assert d[1] == 2
-        e = fromstring('3, 4,5', dtype=uint8, sep=',')
+        e = fromstring('3, 4,5', dtype='uint8', sep=',')
         assert len(e) == 3
         assert e[0] == 3
         assert e[1] == 4
         assert e[2] == 5
-        f = fromstring('\x01\x02\x03\x04\x05', dtype=uint8, count=3)
+        f = fromstring('\x01\x02\x03\x04\x05', dtype='uint8', count=3)
         assert len(f) == 3
         assert f[0] == 1
         assert f[1] == 2
         assert f[2] == 3
-        g = fromstring("1  2    3 ", dtype=uint8, sep=" ")
+        g = fromstring("1  2    3 ", dtype='uint8', sep=" ")
         assert len(g) == 3
         assert g[0] == 1
         assert g[1] == 2
         assert g[2] == 3
-        h = fromstring("1, , 2, 3", dtype=uint8, sep=",")
+        h = fromstring("1, , 2, 3", dtype='uint8', sep=",")
         assert (h == [1, 0, 2, 3]).all()
-        i = fromstring("1    2 3", dtype=uint8, sep=" ")
+        i = fromstring("1    2 3", dtype='uint8', sep=" ")
         assert (i == [1, 2, 3]).all()
-        j = fromstring("1\t\t\t\t2\t3", dtype=uint8, sep="\t")
+        j = fromstring("1\t\t\t\t2\t3", dtype='uint8', sep="\t")
         assert (j == [1, 2, 3]).all()
-        k = fromstring("1,x,2,3", dtype=uint8, sep=",")
+        k = fromstring("1,x,2,3", dtype='uint8', sep=",")
         assert (k == [1, 0]).all()
         l = fromstring("1,x,2,3", dtype='float32', sep=",")
         assert (l == [1.0, -1.0]).all()
         m = fromstring("1,,2,3", sep=",")
         assert (m == [1.0, -1.0, 2.0, 3.0]).all()
-        n = fromstring("3.4 2.0 3.8 2.2", dtype=int32, sep=" ")
+        n = fromstring("3.4 2.0 3.8 2.2", dtype='int32', sep=" ")
         assert (n == [3]).all()
-        o = fromstring("1.0 2f.0f 3.8 2.2", dtype=float32, sep=" ")
+        o = fromstring("1.0 2f.0f 3.8 2.2", dtype='float32', sep=" ")
         assert len(o) == 2
         assert o[0] == 1.0
         assert o[1] == 2.0
@@ -2664,51 +2657,48 @@
             assert (u == [1, 0]).all()
 
     def test_fromstring_types(self):
-        from numpypy import (fromstring, int8, int16, int32, int64, uint8,
-            uint16, uint32, float16, float32, float64, longfloat, array)
-        a = fromstring('\xFF', dtype=int8)
+        from numpypy import fromstring, array, dtype
+        a = fromstring('\xFF', dtype='int8')
         assert a[0] == -1
-        b = fromstring('\xFF', dtype=uint8)
+        b = fromstring('\xFF', dtype='uint8')
         assert b[0] == 255
-        c = fromstring('\xFF\xFF', dtype=int16)
+        c = fromstring('\xFF\xFF', dtype='int16')
         assert c[0] == -1
-        d = fromstring('\xFF\xFF', dtype=uint16)
+        d = fromstring('\xFF\xFF', dtype='uint16')
         assert d[0] == 65535
-        e = fromstring('\xFF\xFF\xFF\xFF', dtype=int32)
+        e = fromstring('\xFF\xFF\xFF\xFF', dtype='int32')
         assert e[0] == -1
-        f = fromstring('\xFF\xFF\xFF\xFF', dtype=uint32)
+        f = fromstring('\xFF\xFF\xFF\xFF', dtype='uint32')
         assert repr(f[0]) == '4294967295'
-        g = fromstring('\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF', dtype=int64)
+        g = fromstring('\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF', dtype='int64')
         assert g[0] == -1
-        h = fromstring(self.float32val, dtype=float32)
-        assert h[0] == float32(5.2)
-        i = fromstring(self.float64val, dtype=float64)
-        assert i[0] == float64(300.4)
+        h = fromstring(self.float32val, dtype='float32')
+        assert h[0] == dtype('float32').type(5.2)
+        i = fromstring(self.float64val, dtype='float64')
+        assert i[0] == dtype('float64').type(300.4)
         j = fromstring(self.ulongval, dtype='L')
         assert j[0] == 12
-        k = fromstring(self.float16val, dtype=float16)
-        assert k[0] == float16(5.)
-        dt =  array([5],dtype=longfloat).dtype
+        k = fromstring(self.float16val, dtype='float16')
+        assert k[0] == dtype('float16').type(5.)
+        dt =  array([5],dtype='longfloat').dtype
         if dt.itemsize == 12:
-            from numpypy import float96
-            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00', dtype=float96)
+            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00', dtype='float96')
         elif dt.itemsize == 16:
-            from numpypy import float128
-            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00\x00\x00\x00\x00', dtype=float128)
+            m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00\x00\x00\x00\x00', dtype='float128')
         elif dt.itemsize == 8:
             skip('longfloat is float64')
         else:
             skip('unknown itemsize for longfloat')
-        assert m[0] == longfloat(5.)
+        assert m[0] == dtype('longfloat').type(5.)
 
     def test_fromstring_invalid(self):
-        from numpypy import fromstring, uint16, uint8
+        from numpypy import fromstring
         #default dtype is 64-bit float, so 3 bytes should fail
         raises(ValueError, fromstring, "\x01\x02\x03")
         #3 bytes is not modulo 2 bytes (int16)
-        raises(ValueError, fromstring, "\x01\x03\x03", dtype=uint16)
+        raises(ValueError, fromstring, "\x01\x03\x03", dtype='uint16')
         #5 bytes is larger than 3 bytes
-        raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype=uint8)
+        raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype='uint8')
 
     def test_tostring(self):
         from numpypy import array
@@ -2747,7 +2737,7 @@
     spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
 
     def test_zeros(self):
-        from numpypy import zeros, integer
+        from numpypy import zeros
         a = zeros(2, dtype=[('x', int), ('y', float)])
         raises(IndexError, 'a[0]["xyz"]')
         assert a[0]['x'] == 0
@@ -2989,8 +2979,8 @@
 
     def test_scalar_coercion(self):
         import numpypy as np
-        a = np.array([1,2,3], dtype=np.int16)
-        assert (a * 2).dtype == np.int16
+        a = np.array([1,2,3], dtype='int16')
+        assert (a * 2).dtype == np.dtype('int16')
 
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -122,6 +122,10 @@
         a = array(range(30))
         assert negative(a + a)[3] == -6
 
+        a = array([[1, 2], [3, 4]])
+        b = negative(a + a)
+        assert (b == [[-2, -4], [-6, -8]]).all()
+
     def test_abs(self):
         from numpypy import array, absolute
 
@@ -244,7 +248,7 @@
         assert isinstance(x, (int, long))
 
     def test_multiply(self):
-        from numpypy import array, multiply
+        from numpypy import array, multiply, arange
 
         a = array([-5.0, -0.0, 1.0])
         b = array([ 3.0, -2.0,-3.0])
@@ -252,6 +256,9 @@
         for i in range(3):
             assert c[i] == a[i] * b[i]
 
+        a = arange(15).reshape(5, 3)
+        assert(multiply.reduce(a) == array([0, 3640, 12320])).all()
+
     def test_rint(self):
         from numpypy import array, complex, rint, isnan
         import sys

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:03 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:03 +0100 (CET)
Subject: [pypy-commit] pypy default: clean up this file used for a single
	function
Message-ID: <20131029193303.6CFE51C1052@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67692:895903d707dd
Date: 2013-10-29 12:00 -0400
http://bitbucket.org/pypy/pypy/changeset/895903d707dd/

Log:	clean up this file used for a single function

diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py
deleted file mode 100644
--- a/pypy/module/micronumpy/dot.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from pypy.interpreter.error import OperationError
-
-def match_dot_shapes(space, left, right):
-    left_shape = left.get_shape()
-    right_shape = right.get_shape()
-    my_critical_dim_size = left_shape[-1]
-    right_critical_dim_size = right_shape[0]
-    right_critical_dim = 0
-    out_shape = []
-    if len(right_shape) > 1:
-        right_critical_dim = len(right_shape) - 2
-        right_critical_dim_size = right_shape[right_critical_dim]
-        assert right_critical_dim >= 0
-        out_shape = out_shape + left_shape[:-1] + \
-                    right_shape[0:right_critical_dim] + \
-                    right_shape[right_critical_dim + 1:]
-    elif len(right_shape) > 0:
-        #dot does not reduce for scalars
-        out_shape = out_shape + left_shape[:-1]
-    if my_critical_dim_size != right_critical_dim_size:
-        raise OperationError(space.w_ValueError, space.wrap(
-                                        "objects are not aligned"))
-    return out_shape, right_critical_dim
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -12,7 +12,6 @@
 from pypy.module.micronumpy.interp_flatiter import W_FlatIterator
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy import loop
-from pypy.module.micronumpy.dot import match_dot_shapes
 from pypy.module.micronumpy.interp_arrayops import repeat, choose, put
 from pypy.module.micronumpy.arrayimpl import scalar
 from rpython.tool.sourcetools import func_with_new_name
@@ -31,6 +30,28 @@
     shape += dtype.shape
     return shape[:]
 
+def _match_dot_shapes(space, left, right):
+    left_shape = left.get_shape()
+    right_shape = right.get_shape()
+    my_critical_dim_size = left_shape[-1]
+    right_critical_dim_size = right_shape[0]
+    right_critical_dim = 0
+    out_shape = []
+    if len(right_shape) > 1:
+        right_critical_dim = len(right_shape) - 2
+        right_critical_dim_size = right_shape[right_critical_dim]
+        assert right_critical_dim >= 0
+        out_shape = out_shape + left_shape[:-1] + \
+                    right_shape[0:right_critical_dim] + \
+                    right_shape[right_critical_dim + 1:]
+    elif len(right_shape) > 0:
+        #dot does not reduce for scalars
+        out_shape = out_shape + left_shape[:-1]
+    if my_critical_dim_size != right_critical_dim_size:
+        raise OperationError(space.w_ValueError, space.wrap(
+                                        "objects are not aligned"))
+    return out_shape, right_critical_dim
+
 class __extend__(W_NDimArray):
     @jit.unroll_safe
     def descr_get_shape(self, space):
@@ -820,7 +841,7 @@
             # numpy compatability
             return W_NDimArray.new_scalar(space, dtype, space.wrap(0))
         # Do the dims match?
-        out_shape, other_critical_dim = match_dot_shapes(space, self, other)
+        out_shape, other_critical_dim = _match_dot_shapes(space, self, other)
         w_res = W_NDimArray.from_shape(space, out_shape, dtype, w_instance=self)
         # This is the place to add fpypy and blas
         return loop.multidim_dot(space, self, other,  w_res, dtype,

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:02 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:02 +0100 (CET)
Subject: [pypy-commit] pypy default: clean up use of numpy constants and
	helper functions
Message-ID: <20131029193302.292551C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67691:72b4a905d844
Date: 2013-10-29 11:57 -0400
http://bitbucket.org/pypy/pypy/changeset/72b4a905d844/

Log:	clean up use of numpy constants and helper functions

diff --git a/pypy/module/micronumpy/constants.py b/pypy/module/micronumpy/constants.py
--- a/pypy/module/micronumpy/constants.py
+++ b/pypy/module/micronumpy/constants.py
@@ -1,21 +1,83 @@
-from pypy.interpreter.error import OperationError
+NPY_BOOL = 0
+NPY_BYTE = 1
+NPY_UBYTE = 2
+NPY_SHORT = 3
+NPY_USHORT = 4
+NPY_INT = 5
+NPY_UINT = 6
+NPY_LONG = 7
+NPY_ULONG = 8
+NPY_LONGLONG = 9
+NPY_ULONGLONG = 10
+NPY_FLOAT = 11
+NPY_DOUBLE = 12
+NPY_LONGDOUBLE = 13
+NPY_CFLOAT = 14
+NPY_CDOUBLE = 15
+NPY_CLONGDOUBLE = 16
+NPY_OBJECT = 17
+NPY_STRING = 18
+NPY_UNICODE = 19
+NPY_VOID = 20
+NPY_DATETIME = 21
+NPY_TIMEDELTA = 22
+NPY_HALF = 23
+NPY_NTYPES = 24
+NPY_NOTYPE = 25
+NPY_CHAR = 26
+NPY_USERDEF = 256
 
-MODE_CLIP, MODE_WRAP, MODE_RAISE = range(3)
+NPY_BOOLLTR = '?'
+NPY_BYTELTR = 'b'
+NPY_UBYTELTR = 'B'
+NPY_SHORTLTR = 'h'
+NPY_USHORTLTR = 'H'
+NPY_INTLTR = 'i'
+NPY_UINTLTR = 'I'
+NPY_LONGLTR = 'l'
+NPY_ULONGLTR = 'L'
+NPY_LONGLONGLTR = 'q'
+NPY_ULONGLONGLTR = 'Q'
+NPY_HALFLTR = 'e'
+NPY_FLOATLTR = 'f'
+NPY_DOUBLELTR = 'd'
+NPY_LONGDOUBLELTR = 'g'
+NPY_CFLOATLTR = 'F'
+NPY_CDOUBLELTR = 'D'
+NPY_CLONGDOUBLELTR = 'G'
+NPY_OBJECTLTR = 'O'
+NPY_STRINGLTR = 'S'
+NPY_STRINGLTR2 = 'a'
+NPY_UNICODELTR = 'U'
+NPY_VOIDLTR = 'V'
+NPY_DATETIMELTR = 'M'
+NPY_TIMEDELTALTR = 'm'
+NPY_CHARLTR = 'c'
 
-def clipmode_converter(space, w_mode):
-    if space.is_none(w_mode):
-        return MODE_RAISE
-    if space.isinstance_w(w_mode, space.w_str):
-        mode = space.str_w(w_mode)
-        if mode.startswith('C') or mode.startswith('c'):
-            return MODE_CLIP
-        if mode.startswith('W') or mode.startswith('w'):
-            return MODE_WRAP
-        if mode.startswith('R') or mode.startswith('r'):
-            return MODE_RAISE
-    elif space.isinstance_w(w_mode, space.w_int):
-        mode = space.int_w(w_mode)
-        if MODE_CLIP <= mode <= MODE_RAISE:
-            return mode
-    raise OperationError(space.w_TypeError,
-                         space.wrap("clipmode not understood"))
+NPY_INTPLTR = 'p'
+NPY_UINTPLTR = 'P'
+
+NPY_GENBOOLLTR ='b'
+NPY_SIGNEDLTR = 'i'
+NPY_UNSIGNEDLTR = 'u'
+NPY_FLOATINGLTR = 'f'
+NPY_COMPLEXLTR = 'c'
+
+NPY_CLIP = 0
+NPY_WRAP = 1
+NPY_RAISE = 2
+
+NPY_LITTLE = '<'
+NPY_BIG = '>'
+NPY_NATIVE = '='
+NPY_SWAP = 's'
+NPY_IGNORE = '|'
+
+import sys
+if sys.byteorder == 'big':
+    NPY_NATBYTE = NPY_BIG
+    NPY_OPPBYTE = NPY_LITTLE
+else:
+    NPY_NATBYTE = NPY_LITTLE
+    NPY_OPPBYTE = NPY_BIG
+del sys
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -1,11 +1,12 @@
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
-from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs, constants
+from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs
 from pypy.module.micronumpy.iter import Chunk, Chunks
 from pypy.module.micronumpy.strides import shape_agreement,\
      shape_agreement_multiple
-from pypy.module.micronumpy.constants import clipmode_converter
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
+from pypy.module.micronumpy.conversion_utils import clipmode_converter
+from pypy.module.micronumpy.constants import *
 
 def where(space, w_arr, w_x=None, w_y=None):
     """where(condition, [x, y])
@@ -219,17 +220,18 @@
         index = int_w(space, idx)
 
         if index < 0 or index >= arr.get_size():
-            if mode == constants.MODE_RAISE:
+            if mode == NPY_RAISE:
                 raise OperationError(space.w_IndexError, space.wrap(
                     "index %d is out of bounds for axis 0 with size %d" % (index, arr.get_size())))
-            elif mode == constants.MODE_WRAP:
+            elif mode == NPY_WRAP:
                 index = index % arr.get_size()
-            else:
-                assert mode == constants.MODE_CLIP
+            elif mode == NPY_CLIP:
                 if index < 0:
                     index = 0
                 else:
                     index = arr.get_size() - 1
+            else:
+                assert False
 
         value = values[v_idx]
 
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -9,24 +9,8 @@
 from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib import jit
+from pypy.module.micronumpy.constants import *
 
-if sys.byteorder == 'little':
-    NATBYTE = '<'
-    OPPBYTE = '>'
-else:
-    NATBYTE = '>'
-    OPPBYTE = '<'
-
-UNSIGNEDLTR = "u"
-SIGNEDLTR = "i"
-BOOLLTR = "b"
-FLOATINGLTR = "f"
-COMPLEXLTR = "c"
-VOIDLTR = 'V'
-STRINGLTR = 'S'
-UNICODELTR = 'U'
-INTPLTR = 'p'
-UINTPLTR = 'P'
 
 def decode_w_dtype(space, w_dtype):
     if space.is_none(w_dtype):
@@ -52,7 +36,7 @@
 class W_Dtype(W_Root):
     _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder", "float_type"]
 
-    def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder='=',
+    def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder=NPY_NATIVE,
                  alternate_constructors=[], aliases=[], float_type=None,
                  fields=None, fieldnames=None, shape=[], subdtype=None):
         self.itemtype = itemtype
@@ -107,35 +91,35 @@
         self.itemtype.fill(storage, self.get_size(), box, start, stop, 0)
 
     def is_int_type(self):
-        return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or
-                self.kind == BOOLLTR)
+        return (self.kind == NPY_SIGNEDLTR or self.kind == NPY_UNSIGNEDLTR or
+                self.kind == NPY_GENBOOLLTR)
 
     def is_signed(self):
-        return self.kind == SIGNEDLTR
+        return self.kind == NPY_SIGNEDLTR
 
     def is_complex_type(self):
-        return self.kind == COMPLEXLTR
+        return self.kind == NPY_COMPLEXLTR
 
     def is_float_type(self):
-        return (self.kind == FLOATINGLTR or self.float_type is not None)
+        return (self.kind == NPY_FLOATINGLTR or self.float_type is not None)
 
     def is_bool_type(self):
-        return self.kind == BOOLLTR
+        return self.kind == NPY_GENBOOLLTR
 
     def is_record_type(self):
         return self.fields is not None
 
     def is_str_type(self):
-        return self.num == 18
+        return self.num == NPY_STRING
 
     def is_str_or_unicode(self):
-        return (self.num == 18 or self.num == 19)
+        return (self.num == NPY_STRING or self.num == NPY_UNICODE)
 
     def is_flexible_type(self):
         return (self.is_str_or_unicode() or self.is_record_type())
 
     def is_native(self):
-        return self.byteorder in ('=', NATBYTE)
+        return self.byteorder in (NPY_NATIVE, NPY_NATBYTE)
 
     def get_size(self):
         return self.itemtype.get_element_size()
@@ -163,15 +147,15 @@
     def descr_get_str(self, space):
         size = self.get_size()
         basic = self.kind
-        if basic == UNICODELTR:
+        if basic == NPY_UNICODELTR:
             size >>= 2
-            endian = NATBYTE
+            endian = NPY_NATBYTE
         elif size <= 1:
-            endian = '|'  # ignore
+            endian = NPY_IGNORE
         else:
             endian = self.byteorder
-            if endian == '=':
-                endian = NATBYTE
+            if endian == NPY_NATIVE:
+                endian = NPY_NATBYTE
         return space.wrap("%s%s%s" % (endian, basic, size))
 
     def descr_get_base(self, space):
@@ -268,7 +252,7 @@
         names = self.descr_get_names(space)
         values = self.descr_get_fields(space)
         if self.fields:
-            endian = '|'
+            endian = NPY_IGNORE
             #TODO: Implement this when subarrays are implemented
             subdescr = space.w_None
             size = 0
@@ -281,8 +265,8 @@
             alignment = space.wrap(1)
         else:
             endian = self.byteorder
-            if endian == '=':
-                endian = NATBYTE
+            if endian == NPY_NATIVE:
+                endian = NPY_NATBYTE
             subdescr = space.w_None
             w_size = space.wrap(-1)
             alignment = space.wrap(-1)
@@ -296,8 +280,8 @@
             raise OperationError(space.w_NotImplementedError, space.wrap("Pickling protocol version not supported"))
 
         endian = space.str_w(space.getitem(w_data, space.wrap(1)))
-        if endian == NATBYTE:
-            endian = '='
+        if endian == NPY_NATBYTE:
+            endian = NPY_NATIVE
         self.byteorder = endian
 
         fieldnames = space.getitem(w_data, space.wrap(3))
@@ -331,8 +315,8 @@
         offset += subdtype.itemtype.get_element_size() * size
         fieldnames.append(fldname)
     itemtype = types.RecordType(ofs_and_items, offset)
-    return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
-                   "V", space.gettypefor(interp_boxes.W_VoidBox), fields=fields,
+    return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
+                   NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), fields=fields,
                    fieldnames=fieldnames)
 
 def dtype_from_dict(space, w_dict):
@@ -358,8 +342,8 @@
             dim = space.int_w(w_dim)
             shape.append(dim)
             size *= dim
-        return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), 20, VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size),
-                    "V", space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype)
+        return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), NPY_VOID, NPY_VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size),
+                    NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype)
 
     if space.is_none(w_dtype):
         return cache.w_float64dtype
@@ -435,26 +419,28 @@
             size = int(name[1:])
         except ValueError:
             raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
-    if char == 'c':
-        char = 'S'
+    if char == NPY_CHARLTR:
+        char = NPY_STRINGLTR
         size = 1
-    if char == 'S':
+
+    if char == NPY_STRINGLTR:
         itemtype = types.StringType(size)
         basename = 'string'
-        num = 18
+        num = NPY_STRING
         w_box_type = space.gettypefor(interp_boxes.W_StringBox)
-    elif char == 'V':
-        num = 20
+    elif char == NPY_VOIDLTR:
+        itemtype = types.VoidType(size)
         basename = 'void'
-        itemtype = types.VoidType(size)
-        return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(size),
-                    "V", space.gettypefor(interp_boxes.W_VoidBox))
+        num = NPY_VOID
+        w_box_type = space.gettypefor(interp_boxes.W_VoidBox)
+    elif char == NPY_UNICODELTR:
+        itemtype = types.UnicodeType(size)
+        basename = 'unicode'
+        num = NPY_UNICODE
+        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
     else:
-        assert char == 'U'
-        basename = 'unicode'
-        itemtype = types.UnicodeType(size)
-        num = 19
-        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
+        assert False
+
     return W_Dtype(itemtype, num, char,
                    basename + str(8 * itemtype.get_element_size()),
                    char, w_box_type)
@@ -463,10 +449,10 @@
     itemtype = types.StringType(size)
     return W_Dtype(
         itemtype,
-        num=18,
-        kind=STRINGLTR,
+        num=NPY_STRING,
+        kind=NPY_STRINGLTR,
         name='string' + str(8 * itemtype.get_element_size()),
-        char='S',
+        char=NPY_STRINGLTR,
         w_box_type = space.gettypefor(interp_boxes.W_StringBox),
     )
 
@@ -474,10 +460,10 @@
     itemtype = types.UnicodeType(size)
     return W_Dtype(
         itemtype,
-        num=19,
-        kind=UNICODELTR,
+        num=NPY_UNICODE,
+        kind=NPY_UNICODELTR,
         name='unicode' + str(8 * itemtype.get_element_size()),
-        char='U',
+        char=NPY_UNICODELTR,
         w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
     )
 
@@ -486,67 +472,67 @@
     def __init__(self, space):
         self.w_booldtype = W_Dtype(
             types.Bool(),
-            num=0,
-            kind=BOOLLTR,
+            num=NPY_BOOL,
+            kind=NPY_GENBOOLLTR,
             name="bool",
-            char="?",
+            char=NPY_BOOLLTR,
             w_box_type=space.gettypefor(interp_boxes.W_BoolBox),
             alternate_constructors=[space.w_bool],
         )
         self.w_int8dtype = W_Dtype(
             types.Int8(),
-            num=1,
-            kind=SIGNEDLTR,
+            num=NPY_BYTE,
+            kind=NPY_SIGNEDLTR,
             name="int8",
-            char="b",
+            char=NPY_BYTELTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int8Box)
         )
         self.w_uint8dtype = W_Dtype(
             types.UInt8(),
-            num=2,
-            kind=UNSIGNEDLTR,
+            num=NPY_UBYTE,
+            kind=NPY_UNSIGNEDLTR,
             name="uint8",
-            char="B",
+            char=NPY_UBYTELTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt8Box),
         )
         self.w_int16dtype = W_Dtype(
             types.Int16(),
-            num=3,
-            kind=SIGNEDLTR,
+            num=NPY_SHORT,
+            kind=NPY_SIGNEDLTR,
             name="int16",
-            char="h",
+            char=NPY_SHORTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int16Box),
         )
         self.w_uint16dtype = W_Dtype(
             types.UInt16(),
-            num=4,
-            kind=UNSIGNEDLTR,
+            num=NPY_USHORT,
+            kind=NPY_UNSIGNEDLTR,
             name="uint16",
-            char="H",
+            char=NPY_USHORTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt16Box),
         )
         self.w_int32dtype = W_Dtype(
             types.Int32(),
-            num=5,
-            kind=SIGNEDLTR,
+            num=NPY_INT,
+            kind=NPY_SIGNEDLTR,
             name="int32",
-            char="i",
+            char=NPY_INTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int32Box),
        )
         self.w_uint32dtype = W_Dtype(
             types.UInt32(),
-            num=6,
-            kind=UNSIGNEDLTR,
+            num=NPY_UINT,
+            kind=NPY_UNSIGNEDLTR,
             name="uint32",
-            char="I",
+            char=NPY_UINTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt32Box),
         )
         self.w_longdtype = W_Dtype(
             types.Long(),
-            num=7,
-            kind=SIGNEDLTR,
+            num=NPY_LONG,
+            kind=NPY_SIGNEDLTR,
             name="int%d" % LONG_BIT,
-            char="l",
+            char=NPY_LONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_LongBox),
             alternate_constructors=[space.w_int,
                                     space.gettypefor(interp_boxes.W_IntegerBox),
@@ -556,10 +542,10 @@
         )
         self.w_ulongdtype = W_Dtype(
             types.ULong(),
-            num=8,
-            kind=UNSIGNEDLTR,
+            num=NPY_ULONG,
+            kind=NPY_UNSIGNEDLTR,
             name="uint%d" % LONG_BIT,
-            char="L",
+            char=NPY_ULONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
             alternate_constructors=[ space.gettypefor(interp_boxes.W_UnsignedIntegerBox),
                                    ],
@@ -567,35 +553,35 @@
         )
         self.w_int64dtype = W_Dtype(
             types.Int64(),
-            num=9,
-            kind=SIGNEDLTR,
+            num=NPY_LONGLONG,
+            kind=NPY_SIGNEDLTR,
             name="int64",
-            char="q",
+            char=NPY_LONGLONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
             alternate_constructors=[space.w_long],
         )
         self.w_uint64dtype = W_Dtype(
             types.UInt64(),
-            num=10,
-            kind=UNSIGNEDLTR,
+            num=NPY_ULONGLONG,
+            kind=NPY_UNSIGNEDLTR,
             name="uint64",
-            char="Q",
+            char=NPY_ULONGLONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt64Box),
         )
         self.w_float32dtype = W_Dtype(
             types.Float32(),
-            num=11,
-            kind=FLOATINGLTR,
+            num=NPY_FLOAT,
+            kind=NPY_FLOATINGLTR,
             name="float32",
-            char="f",
+            char=NPY_FLOATLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Float32Box),
         )
         self.w_float64dtype = W_Dtype(
             types.Float64(),
-            num=12,
-            kind=FLOATINGLTR,
+            num=NPY_DOUBLE,
+            kind=NPY_FLOATINGLTR,
             name="float64",
-            char="d",
+            char=NPY_DOUBLELTR,
             w_box_type = space.gettypefor(interp_boxes.W_Float64Box),
             alternate_constructors=[space.w_float,
                                     space.gettypefor(interp_boxes.W_NumberBox),
@@ -604,28 +590,28 @@
         )
         self.w_floatlongdtype = W_Dtype(
             types.FloatLong(),
-            num=13,
-            kind=FLOATINGLTR,
+            num=NPY_LONGDOUBLE,
+            kind=NPY_FLOATINGLTR,
             name="float%d" % (interp_boxes.long_double_size * 8),
-            char="g",
+            char=NPY_LONGDOUBLELTR,
             w_box_type=space.gettypefor(interp_boxes.W_FloatLongBox),
             aliases=["longdouble", "longfloat"],
         )
         self.w_complex64dtype = W_Dtype(
             types.Complex64(),
-            num=14,
-            kind=COMPLEXLTR,
+            num=NPY_CFLOAT,
+            kind=NPY_COMPLEXLTR,
             name="complex64",
-            char="F",
+            char=NPY_CFLOATLTR,
             w_box_type = space.gettypefor(interp_boxes.W_Complex64Box),
             float_type = self.w_float32dtype,
         )
         self.w_complex128dtype = W_Dtype(
             types.Complex128(),
-            num=15,
-            kind=COMPLEXLTR,
+            num=NPY_CDOUBLE,
+            kind=NPY_COMPLEXLTR,
             name="complex128",
-            char="D",
+            char=NPY_CDOUBLELTR,
             w_box_type = space.gettypefor(interp_boxes.W_Complex128Box),
             alternate_constructors=[space.w_complex],
             aliases=["complex"],
@@ -633,39 +619,39 @@
         )
         self.w_complexlongdtype = W_Dtype(
             types.ComplexLong(),
-            num=16,
-            kind=COMPLEXLTR,
+            num=NPY_CLONGDOUBLE,
+            kind=NPY_COMPLEXLTR,
             name="complex%d" % (interp_boxes.long_double_size * 16),
-            char="G",
+            char=NPY_CLONGDOUBLELTR,
             w_box_type = space.gettypefor(interp_boxes.W_ComplexLongBox),
             aliases=["clongdouble", "clongfloat"],
             float_type = self.w_floatlongdtype,
         )
         self.w_stringdtype = W_Dtype(
             types.StringType(0),
-            num=18,
-            kind=STRINGLTR,
+            num=NPY_STRING,
+            kind=NPY_STRINGLTR,
             name='string',
-            char='S',
+            char=NPY_STRINGLTR,
             w_box_type = space.gettypefor(interp_boxes.W_StringBox),
             alternate_constructors=[space.w_str, space.gettypefor(interp_boxes.W_CharacterBox)],
             aliases=["str"],
         )
         self.w_unicodedtype = W_Dtype(
             types.UnicodeType(0),
-            num=19,
-            kind=UNICODELTR,
+            num=NPY_UNICODE,
+            kind=NPY_UNICODELTR,
             name='unicode',
-            char='U',
+            char=NPY_UNICODELTR,
             w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
             alternate_constructors=[space.w_unicode],
         )
         self.w_voiddtype = W_Dtype(
             types.VoidType(0),
-            num=20,
-            kind=VOIDLTR,
+            num=NPY_VOID,
+            kind=NPY_VOIDLTR,
             name='void',
-            char='V',
+            char=NPY_VOIDLTR,
             w_box_type = space.gettypefor(interp_boxes.W_VoidBox),
             #alternate_constructors=[space.w_buffer],
             # XXX no buffer in space
@@ -674,43 +660,43 @@
         )
         self.w_float16dtype = W_Dtype(
             types.Float16(),
-            num=23,
-            kind=FLOATINGLTR,
+            num=NPY_HALF,
+            kind=NPY_FLOATINGLTR,
             name="float16",
-            char="e",
+            char=NPY_HALFLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Float16Box),
         )
         ptr_size = rffi.sizeof(rffi.CCHARP)
         if ptr_size == 4:
             intp_box = interp_boxes.W_Int32Box
             intp_type = types.Int32()
-            intp_num = 5
+            intp_num = NPY_INT
             uintp_box = interp_boxes.W_UInt32Box
             uintp_type = types.UInt32()
-            uintp_num = 6
+            uintp_num = NPY_UINT
         elif ptr_size == 8:
             intp_box = interp_boxes.W_Int64Box
             intp_type = types.Int64()
-            intp_num = 7
+            intp_num = NPY_LONG
             uintp_box = interp_boxes.W_UInt64Box
             uintp_type = types.UInt64()
-            uintp_num = 8
+            uintp_num = NPY_ULONG
         else:
             raise ValueError('unknown point size %d' % ptr_size)
         self.w_intpdtype = W_Dtype(
             intp_type,
             num=intp_num,
-            kind=INTPLTR,
+            kind=NPY_INTPLTR,
             name='intp',
-            char=INTPLTR,
+            char=NPY_INTPLTR,
             w_box_type = space.gettypefor(intp_box),
         )
         self.w_uintpdtype = W_Dtype(
             uintp_type,
             num=uintp_num,
-            kind=UINTPLTR,
+            kind=NPY_UINTPLTR,
             name='uintp',
-            char=UINTPLTR,
+            char=NPY_UINTPLTR,
             w_box_type = space.gettypefor(uintp_box),
         )
         float_dtypes = [self.w_float16dtype, self.w_float32dtype,
@@ -741,24 +727,24 @@
             self.dtypes_by_name[dtype.name] = dtype
             can_name = dtype.kind + str(dtype.itemtype.get_element_size())
             self.dtypes_by_name[can_name] = dtype
-            self.dtypes_by_name[NATBYTE + can_name] = dtype
-            self.dtypes_by_name['=' + can_name] = dtype
-            new_name = OPPBYTE + can_name
+            self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype
+            self.dtypes_by_name[NPY_NATIVE + can_name] = dtype
+            new_name = NPY_OPPBYTE + can_name
             itemtypename = dtype.itemtype.__class__.__name__
             itemtype = getattr(types, 'NonNative' + itemtypename)()
             self.dtypes_by_name[new_name] = W_Dtype(
                 itemtype,
                 dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                byteorder=OPPBYTE, float_type=dtype.float_type)
+                byteorder=NPY_OPPBYTE, float_type=dtype.float_type)
             if dtype.kind != dtype.char:
                 can_name = dtype.char
-                self.dtypes_by_name[NATBYTE + can_name] = dtype
-                self.dtypes_by_name['=' + can_name] = dtype
-                new_name = OPPBYTE + can_name
+                self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype
+                self.dtypes_by_name[NPY_NATIVE + can_name] = dtype
+                new_name = NPY_OPPBYTE + can_name
                 self.dtypes_by_name[new_name] = W_Dtype(
                     itemtype,
                     dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                    byteorder=OPPBYTE, float_type=dtype.float_type)
+                    byteorder=NPY_OPPBYTE, float_type=dtype.float_type)
 
             for alias in dtype.aliases:
                 self.dtypes_by_name[alias] = dtype
@@ -815,9 +801,9 @@
                        space.wrap(dtype.num),
                        space.wrap(itemsize * 8), # in case of changing
                        # number of bits per byte in the future
-                       space.wrap(itemsize / (2 if dtype.kind == COMPLEXLTR else 1) or 1)]
+                       space.wrap(itemsize / (2 if dtype.kind == NPY_COMPLEXLTR else 1) or 1)]
             if dtype.is_int_type():
-                if dtype.kind == BOOLLTR:
+                if dtype.kind == NPY_GENBOOLLTR:
                     w_maxobj = space.wrap(1)
                     w_minobj = space.wrap(0)
                 elif dtype.is_signed():
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -9,6 +9,7 @@
 from pypy.module.micronumpy.interp_support import unwrap_axis_arg
 from pypy.module.micronumpy.strides import shape_agreement
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
+from pypy.module.micronumpy.constants import *
 
 def done_if_true(dtype, val):
     return dtype.itemtype.bool(val)
@@ -431,16 +432,16 @@
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
     # Some operations promote op(bool, bool) to return int8, rather than bool
-    if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
+    if promote_bools and (dt1.kind == dt2.kind == NPY_GENBOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
 
     # Everything numeric promotes to complex
     if dt2.is_complex_type() or dt1.is_complex_type():
-        if dt2.num == 14:
+        if dt2.num == NPY_CFLOAT:
             return interp_dtype.get_dtype_cache(space).w_complex64dtype
-        elif dt2.num == 15:
+        elif dt2.num == NPY_CDOUBLE:
             return interp_dtype.get_dtype_cache(space).w_complex128dtype
-        elif dt2.num == 16:
+        elif dt2.num == NPY_CLONGDOUBLE:
             return interp_dtype.get_dtype_cache(space).w_complexlongdtype
         else:
             raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
@@ -452,35 +453,30 @@
         return dt2
 
     # Everything promotes to float, and bool promotes to everything.
-    if dt2.kind == interp_dtype.FLOATINGLTR or dt1.kind == interp_dtype.BOOLLTR:
+    if dt2.kind == NPY_FLOATINGLTR or dt1.kind == NPY_GENBOOLLTR:
         # Float32 + 8-bit int = Float64
-        if dt2.num == 11 and dt1.itemtype.get_element_size() >= 4:
+        if dt2.num == NPY_FLOAT and dt1.itemtype.get_element_size() >= 4:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
         return dt2
 
     # for now this means mixing signed and unsigned
-    if dt2.kind == interp_dtype.SIGNEDLTR:
+    if dt2.kind == NPY_SIGNEDLTR:
         # if dt2 has a greater number of bytes, then just go with it
         if dt1.itemtype.get_element_size() < dt2.itemtype.get_element_size():
             return dt2
         # we need to promote both dtypes
         dtypenum = dt2.num + 2
-    elif dt2.num == 10 or (LONG_BIT == 64 and dt2.num == 8):
+    elif dt2.num == NPY_ULONGLONG or (LONG_BIT == 64 and dt2.num == NPY_ULONG):
         # UInt64 + signed = Float64
-        dtypenum = 12
+        dtypenum = NPY_DOUBLE
     elif dt2.is_flexible_type():
         # For those operations that get here (concatenate, stack),
         # flexible types take precedence over numeric type
         if dt2.is_record_type():
             return dt2
         if dt1.is_str_or_unicode():
-            if dt2.num == 18:
-                if dt2.itemtype.get_element_size() >= \
-                           dt1.itemtype.get_element_size():
-                    return dt2
-                return dt1
             if dt2.itemtype.get_element_size() >= \
-                       dt1.itemtype.get_element_size():
+                    dt1.itemtype.get_element_size():
                 return dt2
             return dt1
         return dt2
@@ -490,7 +486,7 @@
     newdtype = interp_dtype.get_dtype_cache(space).dtypes_by_num[dtypenum]
 
     if (newdtype.itemtype.get_element_size() > dt2.itemtype.get_element_size() or
-        newdtype.kind == interp_dtype.FLOATINGLTR):
+        newdtype.kind == NPY_FLOATINGLTR):
         return newdtype
     else:
         # we only promoted to long on 32-bit or to longlong on 64-bit
@@ -501,23 +497,23 @@
 @jit.unroll_safe
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
         promote_bools=False, promote_to_largest=False):
-    if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
+    if promote_bools and (dt.kind == NPY_GENBOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
     if promote_to_float:
-        if dt.kind == interp_dtype.FLOATINGLTR or dt.kind==interp_dtype.COMPLEXLTR:
+        if dt.kind == NPY_FLOATINGLTR or dt.kind == NPY_COMPLEXLTR:
             return dt
-        if dt.num >= 5:
+        if dt.num >= NPY_INT:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
         for bytes, dtype in interp_dtype.get_dtype_cache(space).float_dtypes_by_num_bytes:
-            if (dtype.kind == interp_dtype.FLOATINGLTR and
+            if (dtype.kind == NPY_FLOATINGLTR and
                 dtype.itemtype.get_element_size() > dt.itemtype.get_element_size()):
                 return dtype
     if promote_to_largest:
-        if dt.kind == interp_dtype.BOOLLTR or dt.kind == interp_dtype.SIGNEDLTR:
+        if dt.kind == NPY_GENBOOLLTR or dt.kind == NPY_SIGNEDLTR:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
-        elif dt.kind == interp_dtype.FLOATINGLTR:
+        elif dt.kind == NPY_FLOATINGLTR:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
-        elif dt.kind == interp_dtype.UNSIGNEDLTR:
+        elif dt.kind == NPY_UNSIGNEDLTR:
             return interp_dtype.get_dtype_cache(space).w_uint64dtype
         else:
             assert False
@@ -559,8 +555,8 @@
         if (current_guess is None):
             return interp_dtype.variable_dtype(space,
                                                'S%d' % space.len_w(w_obj))
-        elif current_guess.num ==18:
-            if  current_guess.itemtype.get_size() < space.len_w(w_obj):
+        elif current_guess.num == NPY_STRING:
+            if current_guess.itemtype.get_size() < space.len_w(w_obj):
                 return interp_dtype.variable_dtype(space,
                                                    'S%d' % space.len_w(w_obj))
         return current_guess
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -10,7 +10,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.iter import PureShapeIterator
-from pypy.module.micronumpy import constants
+from pypy.module.micronumpy.constants import *
 from pypy.module.micronumpy.support import int_w
 
 call2_driver = jit.JitDriver(name='numpy_call2',
@@ -583,13 +583,13 @@
                                       mode=mode)
         index = int_w(space, arr_iter.getitem())
         if index < 0 or index >= len(iterators):
-            if mode == constants.MODE_RAISE:
+            if mode == NPY_RAISE:
                 raise OperationError(space.w_ValueError, space.wrap(
                     "invalid entry in choice array"))
-            elif mode == constants.MODE_WRAP:
+            elif mode == NPY_WRAP:
                 index = index % (len(iterators))
             else:
-                assert mode == constants.MODE_CLIP
+                assert mode == NPY_CLIP
                 if index < 0:
                     index = 0
                 else:
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,4 +1,4 @@
-from pypy.module.micronumpy.interp_dtype import NATBYTE, OPPBYTE
+from pypy.module.micronumpy.interp_dtype import NPY_NATBYTE, NPY_OPPBYTE
 from pypy.conftest import option
 import sys
 
@@ -11,5 +11,5 @@
             if '__pypy__' not in sys.builtin_module_names:
                 import numpy
                 sys.modules['numpypy'] = numpy
-        cls.w_non_native_prefix = cls.space.wrap(OPPBYTE)
-        cls.w_native_prefix = cls.space.wrap(NATBYTE)
+        cls.w_non_native_prefix = cls.space.wrap(NPY_OPPBYTE)
+        cls.w_native_prefix = cls.space.wrap(NPY_NATBYTE)

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:04 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:04 +0100 (CET)
Subject: [pypy-commit] pypy default: fix segfault for missing getitem_bool
	on ScalarIterator
Message-ID: <20131029193304.9899A1C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67693:e1753c519765
Date: 2013-10-29 12:11 -0400
http://bitbucket.org/pypy/pypy/changeset/e1753c519765/

Log:	fix segfault for missing getitem_bool on ScalarIterator

diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py
--- a/pypy/module/micronumpy/arrayimpl/scalar.py
+++ b/pypy/module/micronumpy/arrayimpl/scalar.py
@@ -1,4 +1,3 @@
-
 from pypy.module.micronumpy.arrayimpl import base
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy import support
@@ -19,6 +18,9 @@
     def getitem(self):
         return self.v.get_scalar_value()
 
+    def getitem_bool(self):
+        return self.v.dtype.itemtype.bool(self.v.value)
+
     def setitem(self, v):
         self.v.set_scalar_value(v)
 
@@ -181,4 +183,3 @@
     def get_buffer(self, space):
         raise OperationError(space.w_ValueError, space.wrap(
             "cannot point buffer to a scalar"))
-
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1779,6 +1779,15 @@
         raises(IndexError, "arange(10)[array([10])] = 3")
         raises(IndexError, "arange(10)[[-11]] = 3")
 
+    def test_bool_single_index(self):
+        import numpypy as np
+        a = np.array([[1, 2, 3],
+                      [4, 5, 6],
+                      [7, 8, 9]])
+        a[np.array(True)]; skip("broken")  # check for crash but skip rest of test until correct
+        assert (a[np.array(True)] == a[1]).all()
+        assert (a[np.array(False)] == a[0]).all()
+
     def test_bool_array_index(self):
         from numpypy import arange, array
         b = arange(10)

From noreply at buildbot.pypy.org  Tue Oct 29 20:32:58 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:32:58 +0100 (CET)
Subject: [pypy-commit] pypy default: move this where it belongs
Message-ID: <20131029193258.288DC1C0225@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67688:d8fa96108fe6
Date: 2013-10-29 06:07 -0400
http://bitbucket.org/pypy/pypy/changeset/d8fa96108fe6/

Log:	move this where it belongs

diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,6 +1,3 @@
-from pypy.module.micronumpy.interp_dtype import get_dtype_cache
-from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
-        find_unaryop_result_dtype)
 from pypy.module.micronumpy.interp_dtype import NATBYTE, OPPBYTE
 from pypy.conftest import option
 import sys
@@ -10,82 +7,9 @@
 
     @classmethod
     def setup_class(cls):
-        isNumpy = False
         if option.runappdirect:
             if '__pypy__' not in sys.builtin_module_names:
                 import numpy
                 sys.modules['numpypy'] = numpy
-                isNumpy = True
-        cls.w_isNumpy = cls.space.wrap(isNumpy)
         cls.w_non_native_prefix = cls.space.wrap(OPPBYTE)
         cls.w_native_prefix = cls.space.wrap(NATBYTE)
-
-class TestUfuncCoerscion(object):
-    def test_binops(self, space):
-        bool_dtype = get_dtype_cache(space).w_booldtype
-        int8_dtype = get_dtype_cache(space).w_int8dtype
-        int32_dtype = get_dtype_cache(space).w_int32dtype
-        float64_dtype = get_dtype_cache(space).w_float64dtype
-
-        # Basic pairing
-        assert find_binop_result_dtype(space, bool_dtype, bool_dtype) is bool_dtype
-        assert find_binop_result_dtype(space, bool_dtype, float64_dtype) is float64_dtype
-        assert find_binop_result_dtype(space, float64_dtype, bool_dtype) is float64_dtype
-        assert find_binop_result_dtype(space, int32_dtype, int8_dtype) is int32_dtype
-        assert find_binop_result_dtype(space, int32_dtype, bool_dtype) is int32_dtype
-
-        # With promote bool (happens on div), the result is that the op should
-        # promote bools to int8
-        assert find_binop_result_dtype(space, bool_dtype, bool_dtype, promote_bools=True) is int8_dtype
-        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_bools=True) is float64_dtype
-
-        # Coerce to floats
-        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_to_float=True) is float64_dtype
-
-    def test_unaryops(self, space):
-        bool_dtype = get_dtype_cache(space).w_booldtype
-        int8_dtype = get_dtype_cache(space).w_int8dtype
-        uint8_dtype = get_dtype_cache(space).w_uint8dtype
-        int16_dtype = get_dtype_cache(space).w_int16dtype
-        uint16_dtype = get_dtype_cache(space).w_uint16dtype
-        int32_dtype = get_dtype_cache(space).w_int32dtype
-        uint32_dtype = get_dtype_cache(space).w_uint32dtype
-        long_dtype = get_dtype_cache(space).w_longdtype
-        ulong_dtype = get_dtype_cache(space).w_ulongdtype
-        int64_dtype = get_dtype_cache(space).w_int64dtype
-        uint64_dtype = get_dtype_cache(space).w_uint64dtype
-        float16_dtype = get_dtype_cache(space).w_float16dtype
-        float32_dtype = get_dtype_cache(space).w_float32dtype
-        float64_dtype = get_dtype_cache(space).w_float64dtype
-
-        # Normal rules, everything returns itself
-        assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
-        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
-        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
-        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
-        assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
-        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
-        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
-        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
-        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
-        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
-        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
-
-        # Coerce to floats, some of these will eventually be float16, or
-        # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float16_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float16_dtype
-        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float16_dtype
-        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
-        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
-        assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
-
-        # promote bools, happens with sign ufunc
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -222,6 +222,7 @@
         assert b.called_finalize == True
 
     def test___array__(self):
+        import sys
         from numpypy import ndarray, array, dtype
         class D(ndarray):
             def __new__(subtype, shape, dtype):
@@ -239,7 +240,7 @@
         a = C([2, 2], int)
         b = array(a)
         assert b.shape == (2, 2)
-        if not self.isNumpy:
+        if '__pypy__' in sys.builtin_module_names:
             assert b.id == 'subtype'
             assert isinstance(b, D)
         c = array(a, float)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,6 +1,78 @@
-from pypy.conftest import option
-from pypy.interpreter.gateway import interp2app
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
+        find_unaryop_result_dtype)
+from pypy.module.micronumpy.interp_dtype import get_dtype_cache
+
+
+class TestUfuncCoercion(object):
+    def test_binops(self, space):
+        bool_dtype = get_dtype_cache(space).w_booldtype
+        int8_dtype = get_dtype_cache(space).w_int8dtype
+        int32_dtype = get_dtype_cache(space).w_int32dtype
+        float64_dtype = get_dtype_cache(space).w_float64dtype
+
+        # Basic pairing
+        assert find_binop_result_dtype(space, bool_dtype, bool_dtype) is bool_dtype
+        assert find_binop_result_dtype(space, bool_dtype, float64_dtype) is float64_dtype
+        assert find_binop_result_dtype(space, float64_dtype, bool_dtype) is float64_dtype
+        assert find_binop_result_dtype(space, int32_dtype, int8_dtype) is int32_dtype
+        assert find_binop_result_dtype(space, int32_dtype, bool_dtype) is int32_dtype
+
+        # With promote bool (happens on div), the result is that the op should
+        # promote bools to int8
+        assert find_binop_result_dtype(space, bool_dtype, bool_dtype, promote_bools=True) is int8_dtype
+        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_bools=True) is float64_dtype
+
+        # Coerce to floats
+        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_to_float=True) is float64_dtype
+
+    def test_unaryops(self, space):
+        bool_dtype = get_dtype_cache(space).w_booldtype
+        int8_dtype = get_dtype_cache(space).w_int8dtype
+        uint8_dtype = get_dtype_cache(space).w_uint8dtype
+        int16_dtype = get_dtype_cache(space).w_int16dtype
+        uint16_dtype = get_dtype_cache(space).w_uint16dtype
+        int32_dtype = get_dtype_cache(space).w_int32dtype
+        uint32_dtype = get_dtype_cache(space).w_uint32dtype
+        long_dtype = get_dtype_cache(space).w_longdtype
+        ulong_dtype = get_dtype_cache(space).w_ulongdtype
+        int64_dtype = get_dtype_cache(space).w_int64dtype
+        uint64_dtype = get_dtype_cache(space).w_uint64dtype
+        float16_dtype = get_dtype_cache(space).w_float16dtype
+        float32_dtype = get_dtype_cache(space).w_float32dtype
+        float64_dtype = get_dtype_cache(space).w_float64dtype
+
+        # Normal rules, everything returns itself
+        assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
+        assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
+        assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
+        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
+        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
+
+        # Coerce to floats, some of these will eventually be float16, or
+        # whatever our smallest float type is.
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float16_dtype
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float16_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float16_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
+
+        # promote bools, happens with sign ufunc
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
 
 
 class AppTestUfuncs(BaseNumpyAppTest):

From noreply at buildbot.pypy.org  Tue Oct 29 20:32:59 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:32:59 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: merge default
Message-ID: <20131029193259.5563C1C0335@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67689:c9e3c75577a3
Date: 2013-10-29 06:21 -0400
http://bitbucket.org/pypy/pypy/changeset/c9e3c75577a3/

Log:	merge default

diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,6 +1,3 @@
-from pypy.module.micronumpy.interp_dtype import get_dtype_cache
-from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
-        find_unaryop_result_dtype)
 from pypy.module.micronumpy.interp_dtype import NATBYTE, OPPBYTE
 from pypy.conftest import option
 import sys
@@ -10,82 +7,9 @@
 
     @classmethod
     def setup_class(cls):
-        isNumpy = False
         if option.runappdirect:
             if '__pypy__' not in sys.builtin_module_names:
                 import numpy
                 sys.modules['numpypy'] = numpy
-                isNumpy = True
-        cls.w_isNumpy = cls.space.wrap(isNumpy)
         cls.w_non_native_prefix = cls.space.wrap(OPPBYTE)
         cls.w_native_prefix = cls.space.wrap(NATBYTE)
-
-class TestUfuncCoerscion(object):
-    def test_binops(self, space):
-        bool_dtype = get_dtype_cache(space).w_booldtype
-        int8_dtype = get_dtype_cache(space).w_int8dtype
-        int32_dtype = get_dtype_cache(space).w_int32dtype
-        float64_dtype = get_dtype_cache(space).w_float64dtype
-
-        # Basic pairing
-        assert find_binop_result_dtype(space, bool_dtype, bool_dtype) is bool_dtype
-        assert find_binop_result_dtype(space, bool_dtype, float64_dtype) is float64_dtype
-        assert find_binop_result_dtype(space, float64_dtype, bool_dtype) is float64_dtype
-        assert find_binop_result_dtype(space, int32_dtype, int8_dtype) is int32_dtype
-        assert find_binop_result_dtype(space, int32_dtype, bool_dtype) is int32_dtype
-
-        # With promote bool (happens on div), the result is that the op should
-        # promote bools to int8
-        assert find_binop_result_dtype(space, bool_dtype, bool_dtype, promote_bools=True) is int8_dtype
-        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_bools=True) is float64_dtype
-
-        # Coerce to floats
-        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_to_float=True) is float64_dtype
-
-    def test_unaryops(self, space):
-        bool_dtype = get_dtype_cache(space).w_booldtype
-        int8_dtype = get_dtype_cache(space).w_int8dtype
-        uint8_dtype = get_dtype_cache(space).w_uint8dtype
-        int16_dtype = get_dtype_cache(space).w_int16dtype
-        uint16_dtype = get_dtype_cache(space).w_uint16dtype
-        int32_dtype = get_dtype_cache(space).w_int32dtype
-        uint32_dtype = get_dtype_cache(space).w_uint32dtype
-        long_dtype = get_dtype_cache(space).w_longdtype
-        ulong_dtype = get_dtype_cache(space).w_ulongdtype
-        int64_dtype = get_dtype_cache(space).w_int64dtype
-        uint64_dtype = get_dtype_cache(space).w_uint64dtype
-        float16_dtype = get_dtype_cache(space).w_float16dtype
-        float32_dtype = get_dtype_cache(space).w_float32dtype
-        float64_dtype = get_dtype_cache(space).w_float64dtype
-
-        # Normal rules, everything returns itself
-        assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
-        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
-        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
-        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
-        assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
-        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
-        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
-        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
-        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
-        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
-        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
-
-        # Coerce to floats, some of these will eventually be float16, or
-        # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float16_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float16_dtype
-        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float16_dtype
-        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
-        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
-        assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
-
-        # promote bools, happens with sign ufunc
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -222,6 +222,7 @@
         assert b.called_finalize == True
 
     def test___array__(self):
+        import sys
         from numpypy import ndarray, array, dtype
         class D(ndarray):
             def __new__(subtype, shape, dtype):
@@ -239,7 +240,7 @@
         a = C([2, 2], int)
         b = array(a)
         assert b.shape == (2, 2)
-        if not self.isNumpy:
+        if '__pypy__' in sys.builtin_module_names:
             assert b.id == 'subtype'
             assert isinstance(b, D)
         c = array(a, float)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,6 +1,78 @@
-from pypy.conftest import option
-from pypy.interpreter.gateway import interp2app
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
+        find_unaryop_result_dtype)
+from pypy.module.micronumpy.interp_dtype import get_dtype_cache
+
+
+class TestUfuncCoercion(object):
+    def test_binops(self, space):
+        bool_dtype = get_dtype_cache(space).w_booldtype
+        int8_dtype = get_dtype_cache(space).w_int8dtype
+        int32_dtype = get_dtype_cache(space).w_int32dtype
+        float64_dtype = get_dtype_cache(space).w_float64dtype
+
+        # Basic pairing
+        assert find_binop_result_dtype(space, bool_dtype, bool_dtype) is bool_dtype
+        assert find_binop_result_dtype(space, bool_dtype, float64_dtype) is float64_dtype
+        assert find_binop_result_dtype(space, float64_dtype, bool_dtype) is float64_dtype
+        assert find_binop_result_dtype(space, int32_dtype, int8_dtype) is int32_dtype
+        assert find_binop_result_dtype(space, int32_dtype, bool_dtype) is int32_dtype
+
+        # With promote bool (happens on div), the result is that the op should
+        # promote bools to int8
+        assert find_binop_result_dtype(space, bool_dtype, bool_dtype, promote_bools=True) is int8_dtype
+        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_bools=True) is float64_dtype
+
+        # Coerce to floats
+        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_to_float=True) is float64_dtype
+
+    def test_unaryops(self, space):
+        bool_dtype = get_dtype_cache(space).w_booldtype
+        int8_dtype = get_dtype_cache(space).w_int8dtype
+        uint8_dtype = get_dtype_cache(space).w_uint8dtype
+        int16_dtype = get_dtype_cache(space).w_int16dtype
+        uint16_dtype = get_dtype_cache(space).w_uint16dtype
+        int32_dtype = get_dtype_cache(space).w_int32dtype
+        uint32_dtype = get_dtype_cache(space).w_uint32dtype
+        long_dtype = get_dtype_cache(space).w_longdtype
+        ulong_dtype = get_dtype_cache(space).w_ulongdtype
+        int64_dtype = get_dtype_cache(space).w_int64dtype
+        uint64_dtype = get_dtype_cache(space).w_uint64dtype
+        float16_dtype = get_dtype_cache(space).w_float16dtype
+        float32_dtype = get_dtype_cache(space).w_float32dtype
+        float64_dtype = get_dtype_cache(space).w_float64dtype
+
+        # Normal rules, everything returns itself
+        assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
+        assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
+        assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
+        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
+        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
+
+        # Coerce to floats, some of these will eventually be float16, or
+        # whatever our smallest float type is.
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float16_dtype
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float16_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float16_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
+
+        # promote bools, happens with sign ufunc
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
 
 
 class AppTestUfuncs(BaseNumpyAppTest):

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:05 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:05 +0100 (CET)
Subject: [pypy-commit] pypy default: test and fix numpy dtype getitem
	behavior
Message-ID: <20131029193305.C2D521C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67694:00ce0266c6c3
Date: 2013-10-29 12:32 -0400
http://bitbucket.org/pypy/pypy/changeset/00ce0266c6c3/

Log:	test and fix numpy dtype getitem behavior

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -232,14 +232,27 @@
                         raise
                     break
 
-    @unwrap_spec(item=str)
-    def descr_getitem(self, space, item):
+    def descr_getitem(self, space, w_item):
         if self.fields is None:
-            raise OperationError(space.w_KeyError, space.wrap("There are no keys in dtypes %s" % self.name))
+            raise OperationError(space.w_KeyError, space.wrap(
+                "There are no fields in dtype %s." % self.name))
+        if space.isinstance_w(w_item, space.w_basestring):
+            item = space.str_w(w_item)
+        elif space.isinstance_w(w_item, space.w_int):
+            indx = space.int_w(w_item)
+            try:
+                item = self.fieldnames[indx]
+            except IndexError:
+                raise OperationError(space.w_IndexError, space.wrap(
+                    "Field index %d out of range." % indx))
+        else:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "Field key must be an integer, string, or unicode."))
         try:
             return self.fields[item][1]
         except KeyError:
-            raise OperationError(space.w_KeyError, space.wrap("Field named %s not found" % item))
+            raise OperationError(space.w_KeyError, space.wrap(
+                "Field named '%s' not found." % item))
 
     def descr_reduce(self, space):
         w_class = space.type(self)
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -34,6 +34,14 @@
 
         assert dtype(None) is dtype(float)
 
+        e = dtype('int8')
+        exc = raises(KeyError, "e[2]")
+        assert exc.value.message == "There are no fields in dtype int8."
+        exc = raises(KeyError, "e['z']")
+        assert exc.value.message == "There are no fields in dtype int8."
+        exc = raises(KeyError, "e[None]")
+        assert exc.value.message == "There are no fields in dtype int8."
+
         exc = raises(TypeError, dtype, (1, 2))
         assert 'data type not understood' in str(exc.value)
         raises(KeyError, 'dtype(int)["asdasd"]')
@@ -828,7 +836,17 @@
         assert d["x"].itemsize == 16
         e = dtype([("x", "float", 2), ("y", "int", 2)])
         assert e.fields.keys() == keys
-        assert e['x'].shape == (2,)
+        for v in ['x', u'x', 0, -2]:
+            assert e[v] == (dtype('float'), (2,))
+        for v in ['y', u'y', 1, -1]:
+            assert e[v] == (dtype('int'), (2,))
+        for v in [-3, 2]:
+            exc = raises(IndexError, "e[%d]" % v)
+            assert exc.value.message == "Field index %d out of range." % v
+        exc = raises(KeyError, "e['z']")
+        assert exc.value.message == "Field named 'z' not found."
+        exc = raises(ValueError, "e[None]")
+        assert exc.value.message == 'Field key must be an integer, string, or unicode.'
 
         dt = dtype((float, 10))
         assert dt.shape == (10,)

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:00 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:00 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: hack a dummy numpy module for
	testing
Message-ID: <20131029193300.B9F6E1C0930@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67690:ef29b8ac8ebf
Date: 2013-10-29 07:07 -0400
http://bitbucket.org/pypy/pypy/changeset/ef29b8ac8ebf/

Log:	hack a dummy numpy module for testing

diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -0,0 +1,2 @@
+from _numpypy.multiarray import *
+from _numpypy.umath import *
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,6 +1,5 @@
 from pypy.module.micronumpy.interp_dtype import NATBYTE, OPPBYTE
 from pypy.conftest import option
-import sys
 
 class BaseNumpyAppTest(object):
     spaceconfig = dict(usemodules=['micronumpy'])
@@ -8,8 +7,20 @@
     @classmethod
     def setup_class(cls):
         if option.runappdirect:
+            import sys
             if '__pypy__' not in sys.builtin_module_names:
                 import numpy
-                sys.modules['numpypy'] = numpy
+            else:
+                from . import dummy_module as numpy
+            sys.modules['numpypy'] = numpy
+        else:
+            import os
+            path = os.path.dirname(__file__) + '/dummy_module.py'
+            cls.space.appexec([cls.space.wrap(path)], """(path):
+            import imp
+            numpy = imp.load_source('numpy', path)
+            import sys
+            sys.modules['numpypy'] = numpy
+            """)
         cls.w_non_native_prefix = cls.space.wrap(OPPBYTE)
         cls.w_native_prefix = cls.space.wrap(NATBYTE)

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:06 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:06 +0100 (CET)
Subject: [pypy-commit] pypy default: test and fix numpy void getitem behavior
Message-ID: <20131029193306.EC6E11C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67695:be671931b570
Date: 2013-10-29 12:41 -0400
http://bitbucket.org/pypy/pypy/changeset/be671931b570/

Log:	test and fix numpy void getitem behavior

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -378,25 +378,27 @@
 
 class W_VoidBox(W_FlexibleBox):
     def descr_getitem(self, space, w_item):
-        from pypy.module.micronumpy.types import VoidType
-        if space.isinstance_w(w_item, space.w_str):
+        if space.isinstance_w(w_item, space.w_basestring):
             item = space.str_w(w_item)
         elif space.isinstance_w(w_item, space.w_int):
-            #Called by iterator protocol
             indx = space.int_w(w_item)
             try:
                 item = self.dtype.fieldnames[indx]
             except IndexError:
-                raise OperationError(space.w_IndexError,
-                     space.wrap("Iterated over too many fields %d" % indx))
+                if indx < 0:
+                    indx += len(self.dtype.fieldnames)
+                raise OperationError(space.w_IndexError, space.wrap(
+                    "invalid index (%d)" % indx))
         else:
             raise OperationError(space.w_IndexError, space.wrap(
-                    "Can only access fields of record with int or str"))
+                "invalid index"))
         try:
             ofs, dtype = self.dtype.fields[item]
         except KeyError:
-            raise OperationError(space.w_IndexError,
-                                 space.wrap("Field %s does not exist" % item))
+            raise OperationError(space.w_IndexError, space.wrap(
+                "invalid index"))
+
+        from pypy.module.micronumpy.types import VoidType
         if isinstance(dtype.itemtype, VoidType):
             read_val = dtype.itemtype.readarray(self.arr, self.ofs, ofs, dtype)
         else:
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2924,10 +2924,22 @@
         d = dtype([("x", "int", 3), ("y", "float", 5)])
         a = array([([1, 2, 3], [0.5, 1.5, 2.5, 3.5, 4.5]), ([4, 5, 6], [5.5, 6.5, 7.5, 8.5, 9.5])], dtype=d)
 
-        assert (a[0]["x"] == [1, 2, 3]).all()
-        assert (a[0]["y"] == [0.5, 1.5, 2.5, 3.5, 4.5]).all()
-        assert (a[1]["x"] == [4, 5, 6]).all()
-        assert (a[1]["y"] == [5.5, 6.5, 7.5, 8.5, 9.5]).all()
+        for v in ['x', u'x', 0, -2]:
+            assert (a[0][v] == [1, 2, 3]).all()
+            assert (a[1][v] == [4, 5, 6]).all()
+        for v in ['y', u'y', -1, 1]:
+            assert (a[0][v] == [0.5, 1.5, 2.5, 3.5, 4.5]).all()
+            assert (a[1][v] == [5.5, 6.5, 7.5, 8.5, 9.5]).all()
+        for v in [-3, 2]:
+            exc = raises(IndexError, "a[0][%d]" % v)
+            assert exc.value.message == "invalid index (%d)" % (v + 2 if v < 0 else v)
+        exc = raises(IndexError, "a[0]['z']")
+        assert exc.value.message == "invalid index"
+        exc = raises(IndexError, "a[0][None]")
+        assert exc.value.message == "invalid index"
+
+        exc = raises(IndexError, "a[0][None]")
+        assert exc.value.message == 'invalid index'
 
         a[0]["x"][0] = 200
         assert a[0]["x"][0] == 200

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:08 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:08 +0100 (CET)
Subject: [pypy-commit] pypy default: fixes for some numpy exceptions
Message-ID: <20131029193308.1F9D91C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67696:8bbe7e215208
Date: 2013-10-29 13:00 -0400
http://bitbucket.org/pypy/pypy/changeset/8bbe7e215208/

Log:	fixes for some numpy exceptions

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -203,7 +203,7 @@
             dtype = self.dtype
             if not dtype.is_record_type() or idx not in dtype.fields:
                 raise OperationError(space.w_ValueError, space.wrap(
-                    "field named %s not defined" % idx))
+                    "field named %s not found" % idx))
             return RecordChunk(idx)
         if (space.isinstance_w(w_idx, space.w_int) or
             space.isinstance_w(w_idx, space.w_slice)):
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -201,8 +201,6 @@
             return self.implementation.descr_getitem(space, self, w_idx)
         except ArrayArgumentException:
             return self.getitem_array_int(space, w_idx)
-        except OperationError:
-            raise OperationError(space.w_IndexError, space.wrap("wrong index"))
 
     def getitem(self, space, index_list):
         return self.implementation.getitem_index(space, index_list)
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -5,7 +5,11 @@
     try:
         return space.int_w(space.index(w_obj))
     except OperationError:
-        return space.int_w(space.int(w_obj))
+        try:
+            return space.int_w(space.int(w_obj))
+        except OperationError:
+            raise OperationError(space.w_IndexError, space.wrap(
+                "cannot convert index to integer"))
 
 @jit.unroll_safe
 def product(s):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -525,8 +525,10 @@
 
         from numpypy import arange
         a = arange(10)
-        raises(IndexError, "a[ErrorIndex()] == 0")
-        raises(IndexError, "a[ErrorInt()] == 0")
+        exc = raises(IndexError, "a[ErrorIndex()] == 0")
+        assert exc.value.message == 'cannot convert index to integer'
+        exc = raises(IndexError, "a[ErrorInt()] == 0")
+        assert exc.value.message == 'cannot convert index to integer'
 
     def test_setslice_array(self):
         from numpypy import array
@@ -2960,6 +2962,8 @@
         a[0, 0] = 500
         assert (a[0, 0, 0] == 500).all()
         assert a[0, 0, 0].shape == (10,)
+        exc = raises(ValueError, "a[0, 0]['z']")
+        assert exc.value.message == 'field named z not found'
 
     def test_subarray_multiple_rows(self):
         import numpypy as np

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:09 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:09 +0100 (CET)
Subject: [pypy-commit] pypy default: rename numpy support.int_w to index_w
	for clarity
Message-ID: <20131029193309.495C01C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67697:b974175856c4
Date: 2013-10-29 13:08 -0400
http://bitbucket.org/pypy/pypy/changeset/b974175856c4/

Log:	rename numpy support.int_w to index_w for clarity

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -121,7 +121,7 @@
         for i, w_index in enumerate(view_w):
             if space.isinstance_w(w_index, space.w_slice):
                 raise IndexError
-            idx = support.int_w(space, w_index)
+            idx = support.index_w(space, w_index)
             if idx < 0:
                 idx = self.get_shape()[i] + idx
             if idx < 0 or idx >= self.get_shape()[i]:
@@ -193,7 +193,7 @@
             return self._lookup_by_index(space, view_w)
         if shape_len > 1:
             raise IndexError
-        idx = support.int_w(space, w_idx)
+        idx = support.index_w(space, w_idx)
         return self._lookup_by_index(space, [space.wrap(idx)])
 
     @jit.unroll_safe
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -191,7 +191,7 @@
     return out
 
 def put(space, w_arr, w_indices, w_values, w_mode):
-    from pypy.module.micronumpy.support import int_w
+    from pypy.module.micronumpy.support import index_w
 
     arr = convert_to_array(space, w_arr)
     mode = clipmode_converter(space, w_mode)
@@ -217,7 +217,7 @@
 
     v_idx = 0
     for idx in indices:
-        index = int_w(space, idx)
+        index = index_w(space, idx)
 
         if index < 0 or index >= arr.get_size():
             if mode == NPY_RAISE:
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -10,8 +10,8 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.iter import PureShapeIterator
+from pypy.module.micronumpy.support import index_w
 from pypy.module.micronumpy.constants import *
-from pypy.module.micronumpy.support import int_w
 
 call2_driver = jit.JitDriver(name='numpy_call2',
                              greens = ['shapelen', 'func', 'calc_dtype',
@@ -581,7 +581,7 @@
     while not arr_iter.done():
         choose_driver.jit_merge_point(shapelen=shapelen, dtype=dtype,
                                       mode=mode)
-        index = int_w(space, arr_iter.getitem())
+        index = index_w(space, arr_iter.getitem())
         if index < 0 or index >= len(iterators):
             if mode == NPY_RAISE:
                 raise OperationError(space.w_ValueError, space.wrap(
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,7 +1,7 @@
 from rpython.rlib import jit
 from pypy.interpreter.error import OperationError
 
-def int_w(space, w_obj):
+def index_w(space, w_obj):
     try:
         return space.int_w(space.index(w_obj))
     except OperationError:

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:10 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:10 +0100 (CET)
Subject: [pypy-commit] pypy default: provide delitem for ndarray
Message-ID: <20131029193310.771501C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67698:2a3beee2babe
Date: 2013-10-29 13:16 -0400
http://bitbucket.org/pypy/pypy/changeset/2a3beee2babe/

Log:	provide delitem for ndarray

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -218,6 +218,10 @@
         except ArrayArgumentException:
             self.setitem_array_int(space, w_idx, w_value)
 
+    def descr_delitem(self, space, w_idx):
+        raise OperationError(space.w_ValueError, space.wrap(
+            "cannot delete array elements"))
+
     def descr_len(self, space):
         shape = self.get_shape()
         if len(shape):
@@ -1065,6 +1069,7 @@
     __len__ = interp2app(W_NDimArray.descr_len),
     __getitem__ = interp2app(W_NDimArray.descr_getitem),
     __setitem__ = interp2app(W_NDimArray.descr_setitem),
+    __delitem__ = interp2app(W_NDimArray.descr_delitem),
 
     __repr__ = interp2app(W_NDimArray.descr_repr),
     __str__ = interp2app(W_NDimArray.descr_str),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -511,6 +511,12 @@
         a[self.CustomIntObject(1)] = 100
         assert a[1] == 100
 
+    def test_delitem(self):
+        import numpypy as np
+        a = np.arange(10)
+        exc = raises(ValueError, 'del a[2]')
+        assert exc.value.message == 'cannot delete array elements'
+
     def test_access_swallow_exception(self):
         class ErrorIndex(object):
             def __index__(self):

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:11 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:11 +0100 (CET)
Subject: [pypy-commit] pypy default: fix spelling of cumulative attribute
Message-ID: <20131029193311.A04E21C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67699:0fdb3c6fa80c
Date: 2013-10-29 13:22 -0400
http://bitbucket.org/pypy/pypy/changeset/0fdb3c6fa80c/

Log:	fix spelling of cumulative attribute

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -861,7 +861,7 @@
     # ----------------------- reduce -------------------------------
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False,
-                           cumultative=False):
+                           cumulative=False):
         def impl(self, space, w_axis=None, w_dtype=None, w_out=None):
             if space.is_none(w_out):
                 out = None
@@ -872,9 +872,9 @@
                 out = w_out
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(
                 space, self, promote_to_largest, w_axis,
-                False, out, w_dtype, cumultative=cumultative)
+                False, out, w_dtype, cumulative=cumulative)
         return func_with_new_name(impl, "reduce_%s_impl_%d_%d" % (ufunc_name,
-                    promote_to_largest, cumultative))
+                    promote_to_largest, cumulative))
 
     descr_sum = _reduce_ufunc_impl("add")
     descr_sum_promote = _reduce_ufunc_impl("add", True)
@@ -884,8 +884,8 @@
     descr_all = _reduce_ufunc_impl('logical_and')
     descr_any = _reduce_ufunc_impl('logical_or')
 
-    descr_cumsum = _reduce_ufunc_impl('add', cumultative=True)
-    descr_cumprod = _reduce_ufunc_impl('multiply', cumultative=True)
+    descr_cumsum = _reduce_ufunc_impl('add', cumulative=True)
+    descr_cumprod = _reduce_ufunc_impl('multiply', cumulative=True)
 
     def _reduce_argmax_argmin_impl(op_name):
         def impl(self, space, w_axis=None, w_out=None):
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -87,7 +87,7 @@
             out = w_out
         return self.reduce(space, w_obj, False, #do not promote_to_largest
                     w_axis, True, #keepdims must be true
-                    out, w_dtype, cumultative=True)
+                    out, w_dtype, cumulative=True)
 
     @unwrap_spec(skipna=bool, keepdims=bool)
     def descr_reduce(self, space, w_obj, w_axis=None, w_dtype=None,
@@ -159,7 +159,7 @@
                            w_dtype)
 
     def reduce(self, space, w_obj, promote_to_largest, w_axis,
-               keepdims=False, out=None, dtype=None, cumultative=False):
+               keepdims=False, out=None, dtype=None, cumulative=False):
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
                 "supported for binary functions"))
@@ -193,7 +193,7 @@
                                 "%s.reduce without identity", self.name)
         if shapelen > 1 and axis < shapelen:
             temp = None
-            if cumultative:
+            if cumulative:
                 shape = obj_shape[:]
                 temp_shape = obj_shape[:axis] + obj_shape[axis + 1:]
                 if out:
@@ -227,15 +227,15 @@
             else:
                 out = W_NDimArray.from_shape(space, shape, dtype, w_instance=obj)
             return loop.do_axis_reduce(shape, self.func, obj, dtype, axis, out,
-                                       self.identity, cumultative, temp)
-        if cumultative:
+                                       self.identity, cumulative, temp)
+        if cumulative:
             if out:
                 if out.get_shape() != [obj.get_size()]:
                     raise OperationError(space.w_ValueError, space.wrap(
                         "out of incompatible size"))
             else:
                 out = W_NDimArray.from_shape(space, [obj.get_size()], dtype, w_instance=obj)
-            loop.compute_reduce_cumultative(obj, out, dtype, self.func,
+            loop.compute_reduce_cumulative(obj, out, dtype, self.func,
                                             self.identity)
             return out
         if out:
diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py
--- a/pypy/module/micronumpy/iter.py
+++ b/pypy/module/micronumpy/iter.py
@@ -275,11 +275,11 @@
         return self.indexes[d]
 
 class AxisIterator(base.BaseArrayIterator):
-    def __init__(self, array, shape, dim, cumultative):
+    def __init__(self, array, shape, dim, cumulative):
         self.shape = shape
         strides = array.get_strides()
         backstrides = array.get_backstrides()
-        if cumultative:
+        if cumulative:
             self.strides = strides
             self.backstrides = backstrides
         elif len(shape) == len(strides):
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -159,7 +159,7 @@
                                   greens = ['shapelen', 'func', 'dtype'],
                                   reds = 'auto')
 
-def compute_reduce_cumultative(obj, out, calc_dtype, func, identity):
+def compute_reduce_cumulative(obj, out, calc_dtype, func, identity):
     obj_iter = obj.create_iter()
     out_iter = out.create_iter()
     cur_value = identity.convert_to(calc_dtype)
@@ -218,10 +218,10 @@
                                             'func', 'dtype'],
                                     reds='auto')
 
-def do_axis_reduce(shape, func, arr, dtype, axis, out, identity, cumultative,
+def do_axis_reduce(shape, func, arr, dtype, axis, out, identity, cumulative,
                    temp):
-    out_iter = out.create_axis_iter(arr.get_shape(), axis, cumultative)
-    if cumultative:
+    out_iter = out.create_axis_iter(arr.get_shape(), axis, cumulative)
+    if cumulative:
         temp_iter = temp.create_axis_iter(arr.get_shape(), axis, False)
     else:
         temp_iter = out_iter # hack
@@ -240,7 +240,7 @@
             cur = temp_iter.getitem()
             w_val = func(dtype, cur, w_val)
         out_iter.setitem(w_val)
-        if cumultative:
+        if cumulative:
             temp_iter.setitem(w_val)
             temp_iter.next()
         arr_iter.next()

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:12 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:12 +0100 (CET)
Subject: [pypy-commit] pypy default: test and fix for noncommutative
	accumulate segfault
Message-ID: <20131029193312.BDF9A1C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67700:db79497dfb4f
Date: 2013-10-29 13:37 -0400
http://bitbucket.org/pypy/pypy/changeset/db79497dfb4f/

Log:	test and fix for noncommutative accumulate segfault

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -162,7 +162,13 @@
 def compute_reduce_cumulative(obj, out, calc_dtype, func, identity):
     obj_iter = obj.create_iter()
     out_iter = out.create_iter()
-    cur_value = identity.convert_to(calc_dtype)
+    if identity is None:
+        cur_value = obj_iter.getitem().convert_to(calc_dtype)
+        out_iter.setitem(cur_value)
+        out_iter.next()
+        obj_iter.next()
+    else:
+        cur_value = identity.convert_to(calc_dtype)
     shapelen = len(obj.get_shape())
     while not obj_iter.done():
         reduce_cum_driver.jit_merge_point(shapelen=shapelen, func=func,
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1066,3 +1066,14 @@
         print b
         assert (b == [[0, 0, 1], [1, 3, 5]]).all()
         assert b.dtype == int
+
+    def test_noncommutative_reduce_accumulate(self):
+        import numpypy as np
+        tosubtract = np.arange(5)
+        todivide = np.array([2.0, 0.5, 0.25])
+        assert np.subtract.reduce(tosubtract) == -10
+        assert np.divide.reduce(todivide) == 16.0
+        assert (np.subtract.accumulate(tosubtract) ==
+                np.array([0, -1, -3, -6, -10])).all()
+        assert (np.divide.accumulate(todivide) ==
+                np.array([2., 4., 16.])).all()

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:13 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:13 +0100 (CET)
Subject: [pypy-commit] pypy default: fix corner case of signed integer
	division overflow
Message-ID: <20131029193313.EB8DF1C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67701:019951f063bd
Date: 2013-10-29 13:50 -0400
http://bitbucket.org/pypy/pypy/changeset/019951f063bd/

Log:	fix corner case of signed integer division overflow

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1002,6 +1002,13 @@
         b = a // 2
         assert (b == [0, 0, 1, 1, 2]).all()
 
+    def test_signed_integer_division_overflow(self):
+        import numpypy as np
+        for s in (8, 16, 32, 64):
+            for o in ['__div__', '__floordiv__']:
+                a = np.array([-2**(s-1)], dtype='int%d' % s)
+                assert getattr(a, o)(-1) == 0
+
     def test_truediv(self):
         from operator import truediv
         from numpypy import arange
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -422,17 +422,29 @@
     def default_fromstring(self, space):
         return self.box(0)
 
-    @simple_binary_op
-    def div(self, v1, v2):
+    @specialize.argtype(1, 2)
+    def div(self, b1, b2):
+        v1 = self.for_computation(self.unbox(b1))
+        v2 = self.for_computation(self.unbox(b2))
         if v2 == 0:
-            return 0
-        return v1 / v2
+            return self.box(0)
+        if (self.T is rffi.SIGNEDCHAR or self.T is rffi.SHORT or self.T is rffi.INT or
+                self.T is rffi.LONG or self.T is rffi.LONGLONG):
+            if v2 == -1 and v1 == self.for_computation(most_neg_value_of(self.T)):
+                return self.box(0)
+        return self.box(v1 / v2)
 
-    @simple_binary_op
-    def floordiv(self, v1, v2):
+    @specialize.argtype(1, 2)
+    def floordiv(self, b1, b2):
+        v1 = self.for_computation(self.unbox(b1))
+        v2 = self.for_computation(self.unbox(b2))
         if v2 == 0:
-            return 0
-        return v1 // v2
+            return self.box(0)
+        if (self.T is rffi.SIGNEDCHAR or self.T is rffi.SHORT or self.T is rffi.INT or
+                self.T is rffi.LONG or self.T is rffi.LONGLONG):
+            if v2 == -1 and v1 == self.for_computation(most_neg_value_of(self.T)):
+                return self.box(0)
+        return self.box(v1 // v2)
 
     @simple_binary_op
     def mod(self, v1, v2):

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:15 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:15 +0100 (CET)
Subject: [pypy-commit] pypy default: remove NonNativeTypes from micronumpy
Message-ID: <20131029193315.2303F1C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67702:bf1779b754f6
Date: 2013-10-29 14:09 -0400
http://bitbucket.org/pypy/pypy/changeset/bf1779b754f6/

Log:	remove NonNativeTypes from micronumpy

diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -323,8 +323,7 @@
 
 all_types = (types.all_float_types + types.all_complex_types +
              types.all_int_types)
-all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__ and
-                                     not issubclass(i[0], types.BaseFloat16)]
+all_types = [i for i in all_types if not issubclass(i[0], types.Float16)]
 all_types = unrolling_iterable(all_types)
 
 class ArgSortCache(object):
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -743,8 +743,7 @@
             self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype
             self.dtypes_by_name[NPY_NATIVE + can_name] = dtype
             new_name = NPY_OPPBYTE + can_name
-            itemtypename = dtype.itemtype.__class__.__name__
-            itemtype = getattr(types, 'NonNative' + itemtypename)()
+            itemtype = type(dtype.itemtype)(False)
             self.dtypes_by_name[new_name] = W_Dtype(
                 itemtype,
                 dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -116,6 +116,9 @@
     SortRepr = None # placeholders for sorting classes, overloaded in sort.py
     Sort = None
 
+    def __init__(self, native=True):
+        self.native = native
+
     def _unimplemented_ufunc(self, *args):
         raise NotImplementedError
 
@@ -172,7 +175,15 @@
         raise NotImplementedError
 
     def _read(self, storage, i, offset):
-        return raw_storage_getitem(self.T, storage, i + offset)
+        res = raw_storage_getitem(self.T, storage, i + offset)
+        if not self.native:
+            res = byteswap(res)
+        return res
+
+    def _write(self, storage, i, offset, value):
+        if not self.native:
+            value = byteswap(value)
+        raw_storage_setitem(storage, i + offset, value)
 
     def read(self, arr, i, offset, dtype=None):
         return self.box(self._read(arr.storage, i, offset))
@@ -180,9 +191,6 @@
     def read_bool(self, arr, i, offset):
         return bool(self.for_computation(self._read(arr.storage, i, offset)))
 
-    def _write(self, storage, i, offset, value):
-        raw_storage_setitem(storage, i + offset, value)
-
     def store(self, arr, i, offset, box):
         self._write(arr.storage, i, offset, self.unbox(box))
 
@@ -307,17 +315,6 @@
         float64 = Float64()
         return float64.rint(float64.box(v))
 
-class NonNativePrimitive(Primitive):
-    _mixin_ = True
-
-    def _read(self, storage, i, offset):
-        res = raw_storage_getitem(self.T, storage, i + offset)
-        return byteswap(res)
-
-    def _write(self, storage, i, offset, value):
-        value = byteswap(value)
-        raw_storage_setitem(storage, i + offset, value)
-
 class Bool(BaseType, Primitive):
     T = lltype.Bool
     BoxType = interp_boxes.W_BoolBox
@@ -402,8 +399,6 @@
             return 1
         return 0
 
-NonNativeBool = Bool
-
 class Integer(Primitive):
     _mixin_ = True
 
@@ -543,83 +538,46 @@
     def signbit(self, v):
         return v < 0
 
-class NonNativeInteger(NonNativePrimitive, Integer):
-    _mixin_ = True
-
 class Int8(BaseType, Integer):
     T = rffi.SIGNEDCHAR
     BoxType = interp_boxes.W_Int8Box
     format_code = "b"
 
-NonNativeInt8 = Int8
-
 class UInt8(BaseType, Integer):
     T = rffi.UCHAR
     BoxType = interp_boxes.W_UInt8Box
     format_code = "B"
 
-NonNativeUInt8 = UInt8
-
 class Int16(BaseType, Integer):
     T = rffi.SHORT
     BoxType = interp_boxes.W_Int16Box
     format_code = "h"
 
-class NonNativeInt16(BaseType, NonNativeInteger):
-    T = rffi.SHORT
-    BoxType = interp_boxes.W_Int16Box
-    format_code = "h"
-
 class UInt16(BaseType, Integer):
     T = rffi.USHORT
     BoxType = interp_boxes.W_UInt16Box
     format_code = "H"
 
-class NonNativeUInt16(BaseType, NonNativeInteger):
-    T = rffi.USHORT
-    BoxType = interp_boxes.W_UInt16Box
-    format_code = "H"
-
 class Int32(BaseType, Integer):
     T = rffi.INT
     BoxType = interp_boxes.W_Int32Box
     format_code = "i"
 
-class NonNativeInt32(BaseType, NonNativeInteger):
-    T = rffi.INT
-    BoxType = interp_boxes.W_Int32Box
-    format_code = "i"
-
 class UInt32(BaseType, Integer):
     T = rffi.UINT
     BoxType = interp_boxes.W_UInt32Box
     format_code = "I"
 
-class NonNativeUInt32(BaseType, NonNativeInteger):
-    T = rffi.UINT
-    BoxType = interp_boxes.W_UInt32Box
-    format_code = "I"
-
 class Long(BaseType, Integer):
     T = rffi.LONG
     BoxType = interp_boxes.W_LongBox
     format_code = "l"
 
-class NonNativeLong(BaseType, NonNativeInteger):
-    T = rffi.LONG
-    BoxType = interp_boxes.W_LongBox
-    format_code = "l"
-
 class ULong(BaseType, Integer):
     T = rffi.ULONG
     BoxType = interp_boxes.W_ULongBox
     format_code = "L"
 
-class NonNativeULong(BaseType, NonNativeInteger):
-    T = rffi.ULONG
-    BoxType = interp_boxes.W_ULongBox
-    format_code = "L"
-
 def _int64_coerce(self, space, w_item):
     try:
         return self._base_coerce(space, w_item)
@@ -640,13 +598,6 @@
 
     _coerce = func_with_new_name(_int64_coerce, '_coerce')
 
-class NonNativeInt64(BaseType, NonNativeInteger):
-    T = rffi.LONGLONG
-    BoxType = interp_boxes.W_Int64Box
-    format_code = "q"
-
-    _coerce = func_with_new_name(_int64_coerce, '_coerce')
-
 def _uint64_coerce(self, space, w_item):
     try:
         return self._base_coerce(space, w_item)
@@ -667,13 +618,6 @@
 
     _coerce = func_with_new_name(_uint64_coerce, '_coerce')
 
-class NonNativeUInt64(BaseType, NonNativeInteger):
-    T = rffi.ULONGLONG
-    BoxType = interp_boxes.W_UInt64Box
-    format_code = "Q"
-
-    _coerce = func_with_new_name(_uint64_coerce, '_coerce')
-
 class Float(Primitive):
     _mixin_ = True
 
@@ -998,20 +942,7 @@
         else:
             return x
 
-class NonNativeFloat(NonNativePrimitive, Float):
-    _mixin_ = True
-
-    def _read(self, storage, i, offset):
-        res = raw_storage_getitem(self.T, storage, i + offset)
-        return byteswap(res)
-
-    def _write(self, storage, i, offset, value):
-        swapped_value = byteswap(rffi.cast(self.T, value))
-        raw_storage_setitem(storage, i + offset, swapped_value)
-
-class BaseFloat16(Float):
-    _mixin_ = True
-
+class Float16(BaseType, Float):
     _STORAGE_T = rffi.USHORT
     T = rffi.SHORT
     BoxType = interp_boxes.W_Float16Box
@@ -1034,46 +965,29 @@
         swapped = byteswap(rffi.cast(self._STORAGE_T, hbits))
         return self.box(float_unpack(r_ulonglong(swapped), 2))
 
-class Float16(BaseType, BaseFloat16):
     def _read(self, storage, i, offset):
         hbits = raw_storage_getitem(self._STORAGE_T, storage, i + offset)
+        if not self.native:
+            hbits = byteswap(hbits)
         return float_unpack(r_ulonglong(hbits), 2)
 
     def _write(self, storage, i, offset, value):
-        hbits = float_pack(value,2)
+        hbits = rffi.cast(self._STORAGE_T, float_pack(value, 2))
+        if not self.native:
+            hbits = byteswap(hbits)
         raw_storage_setitem(storage, i + offset,
                 rffi.cast(self._STORAGE_T, hbits))
 
-class NonNativeFloat16(BaseType, BaseFloat16):
-    def _read(self, storage, i, offset):
-        hbits = raw_storage_getitem(self._STORAGE_T, storage, i + offset)
-        return float_unpack(r_ulonglong(byteswap(hbits)), 2)
-
-    def _write(self, storage, i, offset, value):
-        hbits = float_pack(value,2)
-        raw_storage_setitem(storage, i + offset,
-                byteswap(rffi.cast(self._STORAGE_T, hbits)))
-
 class Float32(BaseType, Float):
     T = rffi.FLOAT
     BoxType = interp_boxes.W_Float32Box
     format_code = "f"
 
-class NonNativeFloat32(BaseType, NonNativeFloat):
-    T = rffi.FLOAT
-    BoxType = interp_boxes.W_Float32Box
-    format_code = "f"
-
 class Float64(BaseType, Float):
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
     format_code = "d"
 
-class NonNativeFloat64(BaseType, NonNativeFloat):
-    T = rffi.DOUBLE
-    BoxType = interp_boxes.W_Float64Box
-    format_code = "d"
-
 class ComplexFloating(object):
     _mixin_ = True
 
@@ -1625,21 +1539,14 @@
     BoxType = interp_boxes.W_Complex64Box
     ComponentBoxType = interp_boxes.W_Float32Box
 
-NonNativeComplex64 = Complex64
-
 class Complex128(ComplexFloating, BaseType):
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Complex128Box
     ComponentBoxType = interp_boxes.W_Float64Box
 
-NonNativeComplex128 = Complex128
-
 if interp_boxes.long_double_size == 8:
     FloatLong = Float64
-    NonNativeFloatLong = NonNativeFloat64
-
     ComplexLong = Complex128
-    NonNativeComplexLong = NonNativeComplex128
 
 elif interp_boxes.long_double_size in (12, 16):
     class FloatLong(BaseType, Float):
@@ -1657,19 +1564,14 @@
             pack_float80(result, value, 10, not native_is_bigendian)
             return self.box(unpack_float80(result.build(), native_is_bigendian))
 
-    NonNativeFloatLong = FloatLong
-
     class ComplexLong(ComplexFloating, BaseType):
         T = rffi.LONGDOUBLE
         BoxType = interp_boxes.W_ComplexLongBox
         ComponentBoxType = interp_boxes.W_FloatLongBox
 
-    NonNativeComplexLong = ComplexLong
-
-class BaseStringType(object):
-    _mixin_ = True
-
+class BaseStringType(BaseType):
     def __init__(self, size=0):
+        BaseType.__init__(self)
         self.size = size
 
     def get_element_size(self):
@@ -1695,7 +1597,7 @@
         )
     return dispatcher
 
-class StringType(BaseType, BaseStringType):
+class StringType(BaseStringType):
     T = lltype.Char
 
     @jit.unroll_safe
@@ -1812,9 +1714,7 @@
         for i in xrange(start, stop, width):
             self._store(storage, i, offset, box)
 
-NonNativeStringType = StringType
-
-class UnicodeType(BaseType, BaseStringType):
+class UnicodeType(BaseStringType):
     T = lltype.UniChar
 
     @jit.unroll_safe
@@ -1824,9 +1724,7 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "coerce (probably from set_item) not implemented for unicode type"))
 
-NonNativeUnicodeType = UnicodeType
-
-class VoidType(BaseType, BaseStringType):
+class VoidType(BaseStringType):
     T = lltype.Char
 
     def _coerce(self, space, arr, ofs, dtype, w_items, shape):
@@ -1870,12 +1768,11 @@
                              dtype.shape, arr, W_NDimArray(arr), dtype.subdtype)
         return W_NDimArray(implementation)
 
-NonNativeVoidType = VoidType
-
 class RecordType(BaseType):
     T = lltype.Char
 
     def __init__(self, offsets_and_fields, size):
+        BaseType.__init__(self)
         self.offsets_and_fields = offsets_and_fields
         self.size = size
 

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:16 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:16 +0100 (CET)
Subject: [pypy-commit] pypy default: specify _immutable_fields_ for the types
Message-ID: <20131029193316.3D8781C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67703:b85f930a0afa
Date: 2013-10-29 14:11 -0400
http://bitbucket.org/pypy/pypy/changeset/b85f930a0afa/

Log:	specify _immutable_fields_ for the types

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -115,6 +115,7 @@
 class BaseType(object):
     SortRepr = None # placeholders for sorting classes, overloaded in sort.py
     Sort = None
+    _immutable_fields_ = ['native']
 
     def __init__(self, native=True):
         self.native = native
@@ -1570,6 +1571,8 @@
         ComponentBoxType = interp_boxes.W_FloatLongBox
 
 class BaseStringType(BaseType):
+    _immutable_fields = ['size']
+
     def __init__(self, size=0):
         BaseType.__init__(self)
         self.size = size
@@ -1770,6 +1773,7 @@
 
 class RecordType(BaseType):
     T = lltype.Char
+    _immutable_fields_ = ['offsets_and_fields', 'size']
 
     def __init__(self, offsets_and_fields, size):
         BaseType.__init__(self)

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:17 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:17 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: merge default
Message-ID: <20131029193317.8803B1C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67704:b8f2a1ed5e2d
Date: 2013-10-29 15:30 -0400
http://bitbucket.org/pypy/pypy/changeset/b8f2a1ed5e2d/

Log:	merge default

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -121,7 +121,7 @@
         for i, w_index in enumerate(view_w):
             if space.isinstance_w(w_index, space.w_slice):
                 raise IndexError
-            idx = support.int_w(space, w_index)
+            idx = support.index_w(space, w_index)
             if idx < 0:
                 idx = self.get_shape()[i] + idx
             if idx < 0 or idx >= self.get_shape()[i]:
@@ -193,7 +193,7 @@
             return self._lookup_by_index(space, view_w)
         if shape_len > 1:
             raise IndexError
-        idx = support.int_w(space, w_idx)
+        idx = support.index_w(space, w_idx)
         return self._lookup_by_index(space, [space.wrap(idx)])
 
     @jit.unroll_safe
@@ -203,7 +203,7 @@
             dtype = self.dtype
             if not dtype.is_record_type() or idx not in dtype.fields:
                 raise OperationError(space.w_ValueError, space.wrap(
-                    "field named %s not defined" % idx))
+                    "field named %s not found" % idx))
             return RecordChunk(idx)
         if (space.isinstance_w(w_idx, space.w_int) or
             space.isinstance_w(w_idx, space.w_slice)):
diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py
--- a/pypy/module/micronumpy/arrayimpl/scalar.py
+++ b/pypy/module/micronumpy/arrayimpl/scalar.py
@@ -1,4 +1,3 @@
-
 from pypy.module.micronumpy.arrayimpl import base
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy import support
@@ -19,6 +18,9 @@
     def getitem(self):
         return self.v.get_scalar_value()
 
+    def getitem_bool(self):
+        return self.v.dtype.itemtype.bool(self.v.value)
+
     def setitem(self, v):
         self.v.set_scalar_value(v)
 
@@ -181,4 +183,3 @@
     def get_buffer(self, space):
         raise OperationError(space.w_ValueError, space.wrap(
             "cannot point buffer to a scalar"))
-
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -323,8 +323,7 @@
 
 all_types = (types.all_float_types + types.all_complex_types +
              types.all_int_types)
-all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__ and
-                                     not issubclass(i[0], types.BaseFloat16)]
+all_types = [i for i in all_types if not issubclass(i[0], types.Float16)]
 all_types = unrolling_iterable(all_types)
 
 class ArgSortCache(object):
diff --git a/pypy/module/micronumpy/constants.py b/pypy/module/micronumpy/constants.py
--- a/pypy/module/micronumpy/constants.py
+++ b/pypy/module/micronumpy/constants.py
@@ -1,21 +1,83 @@
-from pypy.interpreter.error import OperationError
+NPY_BOOL = 0
+NPY_BYTE = 1
+NPY_UBYTE = 2
+NPY_SHORT = 3
+NPY_USHORT = 4
+NPY_INT = 5
+NPY_UINT = 6
+NPY_LONG = 7
+NPY_ULONG = 8
+NPY_LONGLONG = 9
+NPY_ULONGLONG = 10
+NPY_FLOAT = 11
+NPY_DOUBLE = 12
+NPY_LONGDOUBLE = 13
+NPY_CFLOAT = 14
+NPY_CDOUBLE = 15
+NPY_CLONGDOUBLE = 16
+NPY_OBJECT = 17
+NPY_STRING = 18
+NPY_UNICODE = 19
+NPY_VOID = 20
+NPY_DATETIME = 21
+NPY_TIMEDELTA = 22
+NPY_HALF = 23
+NPY_NTYPES = 24
+NPY_NOTYPE = 25
+NPY_CHAR = 26
+NPY_USERDEF = 256
 
-MODE_CLIP, MODE_WRAP, MODE_RAISE = range(3)
+NPY_BOOLLTR = '?'
+NPY_BYTELTR = 'b'
+NPY_UBYTELTR = 'B'
+NPY_SHORTLTR = 'h'
+NPY_USHORTLTR = 'H'
+NPY_INTLTR = 'i'
+NPY_UINTLTR = 'I'
+NPY_LONGLTR = 'l'
+NPY_ULONGLTR = 'L'
+NPY_LONGLONGLTR = 'q'
+NPY_ULONGLONGLTR = 'Q'
+NPY_HALFLTR = 'e'
+NPY_FLOATLTR = 'f'
+NPY_DOUBLELTR = 'd'
+NPY_LONGDOUBLELTR = 'g'
+NPY_CFLOATLTR = 'F'
+NPY_CDOUBLELTR = 'D'
+NPY_CLONGDOUBLELTR = 'G'
+NPY_OBJECTLTR = 'O'
+NPY_STRINGLTR = 'S'
+NPY_STRINGLTR2 = 'a'
+NPY_UNICODELTR = 'U'
+NPY_VOIDLTR = 'V'
+NPY_DATETIMELTR = 'M'
+NPY_TIMEDELTALTR = 'm'
+NPY_CHARLTR = 'c'
 
-def clipmode_converter(space, w_mode):
-    if space.is_none(w_mode):
-        return MODE_RAISE
-    if space.isinstance_w(w_mode, space.w_str):
-        mode = space.str_w(w_mode)
-        if mode.startswith('C') or mode.startswith('c'):
-            return MODE_CLIP
-        if mode.startswith('W') or mode.startswith('w'):
-            return MODE_WRAP
-        if mode.startswith('R') or mode.startswith('r'):
-            return MODE_RAISE
-    elif space.isinstance_w(w_mode, space.w_int):
-        mode = space.int_w(w_mode)
-        if MODE_CLIP <= mode <= MODE_RAISE:
-            return mode
-    raise OperationError(space.w_TypeError,
-                         space.wrap("clipmode not understood"))
+NPY_INTPLTR = 'p'
+NPY_UINTPLTR = 'P'
+
+NPY_GENBOOLLTR ='b'
+NPY_SIGNEDLTR = 'i'
+NPY_UNSIGNEDLTR = 'u'
+NPY_FLOATINGLTR = 'f'
+NPY_COMPLEXLTR = 'c'
+
+NPY_CLIP = 0
+NPY_WRAP = 1
+NPY_RAISE = 2
+
+NPY_LITTLE = '<'
+NPY_BIG = '>'
+NPY_NATIVE = '='
+NPY_SWAP = 's'
+NPY_IGNORE = '|'
+
+import sys
+if sys.byteorder == 'big':
+    NPY_NATBYTE = NPY_BIG
+    NPY_OPPBYTE = NPY_LITTLE
+else:
+    NPY_NATBYTE = NPY_LITTLE
+    NPY_OPPBYTE = NPY_BIG
+del sys
diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py
deleted file mode 100644
--- a/pypy/module/micronumpy/dot.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from pypy.interpreter.error import OperationError
-
-def match_dot_shapes(space, left, right):
-    left_shape = left.get_shape()
-    right_shape = right.get_shape()
-    my_critical_dim_size = left_shape[-1]
-    right_critical_dim_size = right_shape[0]
-    right_critical_dim = 0
-    out_shape = []
-    if len(right_shape) > 1:
-        right_critical_dim = len(right_shape) - 2
-        right_critical_dim_size = right_shape[right_critical_dim]
-        assert right_critical_dim >= 0
-        out_shape = out_shape + left_shape[:-1] + \
-                    right_shape[0:right_critical_dim] + \
-                    right_shape[right_critical_dim + 1:]
-    elif len(right_shape) > 0:
-        #dot does not reduce for scalars
-        out_shape = out_shape + left_shape[:-1]
-    if my_critical_dim_size != right_critical_dim_size:
-        raise OperationError(space.w_ValueError, space.wrap(
-                                        "objects are not aligned"))
-    return out_shape, right_critical_dim
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -1,11 +1,12 @@
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
-from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs, constants
+from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs
 from pypy.module.micronumpy.iter import Chunk, Chunks
 from pypy.module.micronumpy.strides import shape_agreement,\
      shape_agreement_multiple
-from pypy.module.micronumpy.constants import clipmode_converter
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
+from pypy.module.micronumpy.conversion_utils import clipmode_converter
+from pypy.module.micronumpy.constants import *
 
 def where(space, w_arr, w_x=None, w_y=None):
     """where(condition, [x, y])
@@ -190,7 +191,7 @@
     return out
 
 def put(space, w_arr, w_indices, w_values, w_mode):
-    from pypy.module.micronumpy.support import int_w
+    from pypy.module.micronumpy.support import index_w
 
     arr = convert_to_array(space, w_arr)
     mode = clipmode_converter(space, w_mode)
@@ -216,20 +217,21 @@
 
     v_idx = 0
     for idx in indices:
-        index = int_w(space, idx)
+        index = index_w(space, idx)
 
         if index < 0 or index >= arr.get_size():
-            if mode == constants.MODE_RAISE:
+            if mode == NPY_RAISE:
                 raise OperationError(space.w_IndexError, space.wrap(
                     "index %d is out of bounds for axis 0 with size %d" % (index, arr.get_size())))
-            elif mode == constants.MODE_WRAP:
+            elif mode == NPY_WRAP:
                 index = index % arr.get_size()
-            else:
-                assert mode == constants.MODE_CLIP
+            elif mode == NPY_CLIP:
                 if index < 0:
                     index = 0
                 else:
                     index = arr.get_size() - 1
+            else:
+                assert False
 
         value = values[v_idx]
 
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -378,25 +378,27 @@
 
 class W_VoidBox(W_FlexibleBox):
     def descr_getitem(self, space, w_item):
-        from pypy.module.micronumpy.types import VoidType
-        if space.isinstance_w(w_item, space.w_str):
+        if space.isinstance_w(w_item, space.w_basestring):
             item = space.str_w(w_item)
         elif space.isinstance_w(w_item, space.w_int):
-            #Called by iterator protocol
             indx = space.int_w(w_item)
             try:
                 item = self.dtype.fieldnames[indx]
             except IndexError:
-                raise OperationError(space.w_IndexError,
-                     space.wrap("Iterated over too many fields %d" % indx))
+                if indx < 0:
+                    indx += len(self.dtype.fieldnames)
+                raise OperationError(space.w_IndexError, space.wrap(
+                    "invalid index (%d)" % indx))
         else:
             raise OperationError(space.w_IndexError, space.wrap(
-                    "Can only access fields of record with int or str"))
+                "invalid index"))
         try:
             ofs, dtype = self.dtype.fields[item]
         except KeyError:
-            raise OperationError(space.w_IndexError,
-                                 space.wrap("Field %s does not exist" % item))
+            raise OperationError(space.w_IndexError, space.wrap(
+                "invalid index"))
+
+        from pypy.module.micronumpy.types import VoidType
         if isinstance(dtype.itemtype, VoidType):
             read_val = dtype.itemtype.readarray(self.arr, self.ofs, ofs, dtype)
         else:
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -9,24 +9,8 @@
 from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib import jit
+from pypy.module.micronumpy.constants import *
 
-if sys.byteorder == 'little':
-    NATBYTE = '<'
-    OPPBYTE = '>'
-else:
-    NATBYTE = '>'
-    OPPBYTE = '<'
-
-UNSIGNEDLTR = "u"
-SIGNEDLTR = "i"
-BOOLLTR = "b"
-FLOATINGLTR = "f"
-COMPLEXLTR = "c"
-VOIDLTR = 'V'
-STRINGLTR = 'S'
-UNICODELTR = 'U'
-INTPLTR = 'p'
-UINTPLTR = 'P'
 
 def decode_w_dtype(space, w_dtype):
     if space.is_none(w_dtype):
@@ -52,7 +36,7 @@
 class W_Dtype(W_Root):
     _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder", "float_type"]
 
-    def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder='=',
+    def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder=NPY_NATIVE,
                  alternate_constructors=[], aliases=[], float_type=None,
                  fields=None, fieldnames=None, shape=[], subdtype=None):
         self.itemtype = itemtype
@@ -107,35 +91,35 @@
         self.itemtype.fill(storage, self.get_size(), box, start, stop, 0)
 
     def is_int_type(self):
-        return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or
-                self.kind == BOOLLTR)
+        return (self.kind == NPY_SIGNEDLTR or self.kind == NPY_UNSIGNEDLTR or
+                self.kind == NPY_GENBOOLLTR)
 
     def is_signed(self):
-        return self.kind == SIGNEDLTR
+        return self.kind == NPY_SIGNEDLTR
 
     def is_complex_type(self):
-        return self.kind == COMPLEXLTR
+        return self.kind == NPY_COMPLEXLTR
 
     def is_float_type(self):
-        return (self.kind == FLOATINGLTR or self.float_type is not None)
+        return (self.kind == NPY_FLOATINGLTR or self.float_type is not None)
 
     def is_bool_type(self):
-        return self.kind == BOOLLTR
+        return self.kind == NPY_GENBOOLLTR
 
     def is_record_type(self):
         return self.fields is not None
 
     def is_str_type(self):
-        return self.num == 18
+        return self.num == NPY_STRING
 
     def is_str_or_unicode(self):
-        return (self.num == 18 or self.num == 19)
+        return (self.num == NPY_STRING or self.num == NPY_UNICODE)
 
     def is_flexible_type(self):
         return (self.is_str_or_unicode() or self.is_record_type())
 
     def is_native(self):
-        return self.byteorder in ('=', NATBYTE)
+        return self.byteorder in (NPY_NATIVE, NPY_NATBYTE)
 
     def get_size(self):
         return self.itemtype.get_element_size()
@@ -163,15 +147,15 @@
     def descr_get_str(self, space):
         size = self.get_size()
         basic = self.kind
-        if basic == UNICODELTR:
+        if basic == NPY_UNICODELTR:
             size >>= 2
-            endian = NATBYTE
+            endian = NPY_NATBYTE
         elif size <= 1:
-            endian = '|'  # ignore
+            endian = NPY_IGNORE
         else:
             endian = self.byteorder
-            if endian == '=':
-                endian = NATBYTE
+            if endian == NPY_NATIVE:
+                endian = NPY_NATBYTE
         return space.wrap("%s%s%s" % (endian, basic, size))
 
     def descr_get_base(self, space):
@@ -248,14 +232,27 @@
                         raise
                     break
 
-    @unwrap_spec(item=str)
-    def descr_getitem(self, space, item):
+    def descr_getitem(self, space, w_item):
         if self.fields is None:
-            raise OperationError(space.w_KeyError, space.wrap("There are no keys in dtypes %s" % self.name))
+            raise OperationError(space.w_KeyError, space.wrap(
+                "There are no fields in dtype %s." % self.name))
+        if space.isinstance_w(w_item, space.w_basestring):
+            item = space.str_w(w_item)
+        elif space.isinstance_w(w_item, space.w_int):
+            indx = space.int_w(w_item)
+            try:
+                item = self.fieldnames[indx]
+            except IndexError:
+                raise OperationError(space.w_IndexError, space.wrap(
+                    "Field index %d out of range." % indx))
+        else:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "Field key must be an integer, string, or unicode."))
         try:
             return self.fields[item][1]
         except KeyError:
-            raise OperationError(space.w_KeyError, space.wrap("Field named %s not found" % item))
+            raise OperationError(space.w_KeyError, space.wrap(
+                "Field named '%s' not found." % item))
 
     def descr_reduce(self, space):
         w_class = space.type(self)
@@ -268,7 +265,7 @@
         names = self.descr_get_names(space)
         values = self.descr_get_fields(space)
         if self.fields:
-            endian = '|'
+            endian = NPY_IGNORE
             #TODO: Implement this when subarrays are implemented
             subdescr = space.w_None
             size = 0
@@ -281,8 +278,8 @@
             alignment = space.wrap(1)
         else:
             endian = self.byteorder
-            if endian == '=':
-                endian = NATBYTE
+            if endian == NPY_NATIVE:
+                endian = NPY_NATBYTE
             subdescr = space.w_None
             w_size = space.wrap(-1)
             alignment = space.wrap(-1)
@@ -296,8 +293,8 @@
             raise OperationError(space.w_NotImplementedError, space.wrap("Pickling protocol version not supported"))
 
         endian = space.str_w(space.getitem(w_data, space.wrap(1)))
-        if endian == NATBYTE:
-            endian = '='
+        if endian == NPY_NATBYTE:
+            endian = NPY_NATIVE
         self.byteorder = endian
 
         fieldnames = space.getitem(w_data, space.wrap(3))
@@ -331,8 +328,8 @@
         offset += subdtype.itemtype.get_element_size() * size
         fieldnames.append(fldname)
     itemtype = types.RecordType(ofs_and_items, offset)
-    return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
-                   "V", space.gettypefor(interp_boxes.W_VoidBox), fields=fields,
+    return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
+                   NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), fields=fields,
                    fieldnames=fieldnames)
 
 def dtype_from_dict(space, w_dict):
@@ -358,8 +355,8 @@
             dim = space.int_w(w_dim)
             shape.append(dim)
             size *= dim
-        return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), 20, VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size),
-                    "V", space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype)
+        return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), NPY_VOID, NPY_VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size),
+                    NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype)
 
     if space.is_none(w_dtype):
         return cache.w_float64dtype
@@ -435,26 +432,28 @@
             size = int(name[1:])
         except ValueError:
             raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
-    if char == 'c':
-        char = 'S'
+    if char == NPY_CHARLTR:
+        char = NPY_STRINGLTR
         size = 1
-    if char == 'S':
+
+    if char == NPY_STRINGLTR:
         itemtype = types.StringType(size)
         basename = 'string'
-        num = 18
+        num = NPY_STRING
         w_box_type = space.gettypefor(interp_boxes.W_StringBox)
-    elif char == 'V':
-        num = 20
+    elif char == NPY_VOIDLTR:
+        itemtype = types.VoidType(size)
         basename = 'void'
-        itemtype = types.VoidType(size)
-        return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(size),
-                    "V", space.gettypefor(interp_boxes.W_VoidBox))
+        num = NPY_VOID
+        w_box_type = space.gettypefor(interp_boxes.W_VoidBox)
+    elif char == NPY_UNICODELTR:
+        itemtype = types.UnicodeType(size)
+        basename = 'unicode'
+        num = NPY_UNICODE
+        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
     else:
-        assert char == 'U'
-        basename = 'unicode'
-        itemtype = types.UnicodeType(size)
-        num = 19
-        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
+        assert False
+
     return W_Dtype(itemtype, num, char,
                    basename + str(8 * itemtype.get_element_size()),
                    char, w_box_type)
@@ -463,10 +462,10 @@
     itemtype = types.StringType(size)
     return W_Dtype(
         itemtype,
-        num=18,
-        kind=STRINGLTR,
+        num=NPY_STRING,
+        kind=NPY_STRINGLTR,
         name='string' + str(8 * itemtype.get_element_size()),
-        char='S',
+        char=NPY_STRINGLTR,
         w_box_type = space.gettypefor(interp_boxes.W_StringBox),
     )
 
@@ -474,10 +473,10 @@
     itemtype = types.UnicodeType(size)
     return W_Dtype(
         itemtype,
-        num=19,
-        kind=UNICODELTR,
+        num=NPY_UNICODE,
+        kind=NPY_UNICODELTR,
         name='unicode' + str(8 * itemtype.get_element_size()),
-        char='U',
+        char=NPY_UNICODELTR,
         w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
     )
 
@@ -486,67 +485,67 @@
     def __init__(self, space):
         self.w_booldtype = W_Dtype(
             types.Bool(),
-            num=0,
-            kind=BOOLLTR,
+            num=NPY_BOOL,
+            kind=NPY_GENBOOLLTR,
             name="bool",
-            char="?",
+            char=NPY_BOOLLTR,
             w_box_type=space.gettypefor(interp_boxes.W_BoolBox),
             alternate_constructors=[space.w_bool],
         )
         self.w_int8dtype = W_Dtype(
             types.Int8(),
-            num=1,
-            kind=SIGNEDLTR,
+            num=NPY_BYTE,
+            kind=NPY_SIGNEDLTR,
             name="int8",
-            char="b",
+            char=NPY_BYTELTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int8Box)
         )
         self.w_uint8dtype = W_Dtype(
             types.UInt8(),
-            num=2,
-            kind=UNSIGNEDLTR,
+            num=NPY_UBYTE,
+            kind=NPY_UNSIGNEDLTR,
             name="uint8",
-            char="B",
+            char=NPY_UBYTELTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt8Box),
         )
         self.w_int16dtype = W_Dtype(
             types.Int16(),
-            num=3,
-            kind=SIGNEDLTR,
+            num=NPY_SHORT,
+            kind=NPY_SIGNEDLTR,
             name="int16",
-            char="h",
+            char=NPY_SHORTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int16Box),
         )
         self.w_uint16dtype = W_Dtype(
             types.UInt16(),
-            num=4,
-            kind=UNSIGNEDLTR,
+            num=NPY_USHORT,
+            kind=NPY_UNSIGNEDLTR,
             name="uint16",
-            char="H",
+            char=NPY_USHORTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt16Box),
         )
         self.w_int32dtype = W_Dtype(
             types.Int32(),
-            num=5,
-            kind=SIGNEDLTR,
+            num=NPY_INT,
+            kind=NPY_SIGNEDLTR,
             name="int32",
-            char="i",
+            char=NPY_INTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int32Box),
        )
         self.w_uint32dtype = W_Dtype(
             types.UInt32(),
-            num=6,
-            kind=UNSIGNEDLTR,
+            num=NPY_UINT,
+            kind=NPY_UNSIGNEDLTR,
             name="uint32",
-            char="I",
+            char=NPY_UINTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt32Box),
         )
         self.w_longdtype = W_Dtype(
             types.Long(),
-            num=7,
-            kind=SIGNEDLTR,
+            num=NPY_LONG,
+            kind=NPY_SIGNEDLTR,
             name="int%d" % LONG_BIT,
-            char="l",
+            char=NPY_LONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_LongBox),
             alternate_constructors=[space.w_int,
                                     space.gettypefor(interp_boxes.W_IntegerBox),
@@ -556,10 +555,10 @@
         )
         self.w_ulongdtype = W_Dtype(
             types.ULong(),
-            num=8,
-            kind=UNSIGNEDLTR,
+            num=NPY_ULONG,
+            kind=NPY_UNSIGNEDLTR,
             name="uint%d" % LONG_BIT,
-            char="L",
+            char=NPY_ULONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
             alternate_constructors=[ space.gettypefor(interp_boxes.W_UnsignedIntegerBox),
                                    ],
@@ -567,35 +566,35 @@
         )
         self.w_int64dtype = W_Dtype(
             types.Int64(),
-            num=9,
-            kind=SIGNEDLTR,
+            num=NPY_LONGLONG,
+            kind=NPY_SIGNEDLTR,
             name="int64",
-            char="q",
+            char=NPY_LONGLONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
             alternate_constructors=[space.w_long],
         )
         self.w_uint64dtype = W_Dtype(
             types.UInt64(),
-            num=10,
-            kind=UNSIGNEDLTR,
+            num=NPY_ULONGLONG,
+            kind=NPY_UNSIGNEDLTR,
             name="uint64",
-            char="Q",
+            char=NPY_ULONGLONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt64Box),
         )
         self.w_float32dtype = W_Dtype(
             types.Float32(),
-            num=11,
-            kind=FLOATINGLTR,
+            num=NPY_FLOAT,
+            kind=NPY_FLOATINGLTR,
             name="float32",
-            char="f",
+            char=NPY_FLOATLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Float32Box),
         )
         self.w_float64dtype = W_Dtype(
             types.Float64(),
-            num=12,
-            kind=FLOATINGLTR,
+            num=NPY_DOUBLE,
+            kind=NPY_FLOATINGLTR,
             name="float64",
-            char="d",
+            char=NPY_DOUBLELTR,
             w_box_type = space.gettypefor(interp_boxes.W_Float64Box),
             alternate_constructors=[space.w_float,
                                     space.gettypefor(interp_boxes.W_NumberBox),
@@ -604,28 +603,28 @@
         )
         self.w_floatlongdtype = W_Dtype(
             types.FloatLong(),
-            num=13,
-            kind=FLOATINGLTR,
+            num=NPY_LONGDOUBLE,
+            kind=NPY_FLOATINGLTR,
             name="float%d" % (interp_boxes.long_double_size * 8),
-            char="g",
+            char=NPY_LONGDOUBLELTR,
             w_box_type=space.gettypefor(interp_boxes.W_FloatLongBox),
             aliases=["longdouble", "longfloat"],
         )
         self.w_complex64dtype = W_Dtype(
             types.Complex64(),
-            num=14,
-            kind=COMPLEXLTR,
+            num=NPY_CFLOAT,
+            kind=NPY_COMPLEXLTR,
             name="complex64",
-            char="F",
+            char=NPY_CFLOATLTR,
             w_box_type = space.gettypefor(interp_boxes.W_Complex64Box),
             float_type = self.w_float32dtype,
         )
         self.w_complex128dtype = W_Dtype(
             types.Complex128(),
-            num=15,
-            kind=COMPLEXLTR,
+            num=NPY_CDOUBLE,
+            kind=NPY_COMPLEXLTR,
             name="complex128",
-            char="D",
+            char=NPY_CDOUBLELTR,
             w_box_type = space.gettypefor(interp_boxes.W_Complex128Box),
             alternate_constructors=[space.w_complex],
             aliases=["complex"],
@@ -633,39 +632,39 @@
         )
         self.w_complexlongdtype = W_Dtype(
             types.ComplexLong(),
-            num=16,
-            kind=COMPLEXLTR,
+            num=NPY_CLONGDOUBLE,
+            kind=NPY_COMPLEXLTR,
             name="complex%d" % (interp_boxes.long_double_size * 16),
-            char="G",
+            char=NPY_CLONGDOUBLELTR,
             w_box_type = space.gettypefor(interp_boxes.W_ComplexLongBox),
             aliases=["clongdouble", "clongfloat"],
             float_type = self.w_floatlongdtype,
         )
         self.w_stringdtype = W_Dtype(
             types.StringType(0),
-            num=18,
-            kind=STRINGLTR,
+            num=NPY_STRING,
+            kind=NPY_STRINGLTR,
             name='string',
-            char='S',
+            char=NPY_STRINGLTR,
             w_box_type = space.gettypefor(interp_boxes.W_StringBox),
             alternate_constructors=[space.w_str, space.gettypefor(interp_boxes.W_CharacterBox)],
             aliases=["str"],
         )
         self.w_unicodedtype = W_Dtype(
             types.UnicodeType(0),
-            num=19,
-            kind=UNICODELTR,
+            num=NPY_UNICODE,
+            kind=NPY_UNICODELTR,
             name='unicode',
-            char='U',
+            char=NPY_UNICODELTR,
             w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
             alternate_constructors=[space.w_unicode],
         )
         self.w_voiddtype = W_Dtype(
             types.VoidType(0),
-            num=20,
-            kind=VOIDLTR,
+            num=NPY_VOID,
+            kind=NPY_VOIDLTR,
             name='void',
-            char='V',
+            char=NPY_VOIDLTR,
             w_box_type = space.gettypefor(interp_boxes.W_VoidBox),
             #alternate_constructors=[space.w_buffer],
             # XXX no buffer in space
@@ -674,43 +673,43 @@
         )
         self.w_float16dtype = W_Dtype(
             types.Float16(),
-            num=23,
-            kind=FLOATINGLTR,
+            num=NPY_HALF,
+            kind=NPY_FLOATINGLTR,
             name="float16",
-            char="e",
+            char=NPY_HALFLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Float16Box),
         )
         ptr_size = rffi.sizeof(rffi.CCHARP)
         if ptr_size == 4:
             intp_box = interp_boxes.W_Int32Box
             intp_type = types.Int32()
-            intp_num = 5
+            intp_num = NPY_INT
             uintp_box = interp_boxes.W_UInt32Box
             uintp_type = types.UInt32()
-            uintp_num = 6
+            uintp_num = NPY_UINT
         elif ptr_size == 8:
             intp_box = interp_boxes.W_Int64Box
             intp_type = types.Int64()
-            intp_num = 7
+            intp_num = NPY_LONG
             uintp_box = interp_boxes.W_UInt64Box
             uintp_type = types.UInt64()
-            uintp_num = 8
+            uintp_num = NPY_ULONG
         else:
             raise ValueError('unknown point size %d' % ptr_size)
         self.w_intpdtype = W_Dtype(
             intp_type,
             num=intp_num,
-            kind=INTPLTR,
+            kind=NPY_INTPLTR,
             name='intp',
-            char=INTPLTR,
+            char=NPY_INTPLTR,
             w_box_type = space.gettypefor(intp_box),
         )
         self.w_uintpdtype = W_Dtype(
             uintp_type,
             num=uintp_num,
-            kind=UINTPLTR,
+            kind=NPY_UINTPLTR,
             name='uintp',
-            char=UINTPLTR,
+            char=NPY_UINTPLTR,
             w_box_type = space.gettypefor(uintp_box),
         )
         float_dtypes = [self.w_float16dtype, self.w_float32dtype,
@@ -741,24 +740,23 @@
             self.dtypes_by_name[dtype.name] = dtype
             can_name = dtype.kind + str(dtype.itemtype.get_element_size())
             self.dtypes_by_name[can_name] = dtype
-            self.dtypes_by_name[NATBYTE + can_name] = dtype
-            self.dtypes_by_name['=' + can_name] = dtype
-            new_name = OPPBYTE + can_name
-            itemtypename = dtype.itemtype.__class__.__name__
-            itemtype = getattr(types, 'NonNative' + itemtypename)()
+            self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype
+            self.dtypes_by_name[NPY_NATIVE + can_name] = dtype
+            new_name = NPY_OPPBYTE + can_name
+            itemtype = type(dtype.itemtype)(False)
             self.dtypes_by_name[new_name] = W_Dtype(
                 itemtype,
                 dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                byteorder=OPPBYTE, float_type=dtype.float_type)
+                byteorder=NPY_OPPBYTE, float_type=dtype.float_type)
             if dtype.kind != dtype.char:
                 can_name = dtype.char
-                self.dtypes_by_name[NATBYTE + can_name] = dtype
-                self.dtypes_by_name['=' + can_name] = dtype
-                new_name = OPPBYTE + can_name
+                self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype
+                self.dtypes_by_name[NPY_NATIVE + can_name] = dtype
+                new_name = NPY_OPPBYTE + can_name
                 self.dtypes_by_name[new_name] = W_Dtype(
                     itemtype,
                     dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                    byteorder=OPPBYTE, float_type=dtype.float_type)
+                    byteorder=NPY_OPPBYTE, float_type=dtype.float_type)
 
             for alias in dtype.aliases:
                 self.dtypes_by_name[alias] = dtype
@@ -815,9 +813,9 @@
                        space.wrap(dtype.num),
                        space.wrap(itemsize * 8), # in case of changing
                        # number of bits per byte in the future
-                       space.wrap(itemsize / (2 if dtype.kind == COMPLEXLTR else 1) or 1)]
+                       space.wrap(itemsize / (2 if dtype.kind == NPY_COMPLEXLTR else 1) or 1)]
             if dtype.is_int_type():
-                if dtype.kind == BOOLLTR:
+                if dtype.kind == NPY_GENBOOLLTR:
                     w_maxobj = space.wrap(1)
                     w_minobj = space.wrap(0)
                 elif dtype.is_signed():
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -12,7 +12,6 @@
 from pypy.module.micronumpy.interp_flatiter import W_FlatIterator
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy import loop
-from pypy.module.micronumpy.dot import match_dot_shapes
 from pypy.module.micronumpy.interp_arrayops import repeat, choose, put
 from pypy.module.micronumpy.arrayimpl import scalar
 from rpython.tool.sourcetools import func_with_new_name
@@ -31,6 +30,28 @@
     shape += dtype.shape
     return shape[:]
 
+def _match_dot_shapes(space, left, right):
+    left_shape = left.get_shape()
+    right_shape = right.get_shape()
+    my_critical_dim_size = left_shape[-1]
+    right_critical_dim_size = right_shape[0]
+    right_critical_dim = 0
+    out_shape = []
+    if len(right_shape) > 1:
+        right_critical_dim = len(right_shape) - 2
+        right_critical_dim_size = right_shape[right_critical_dim]
+        assert right_critical_dim >= 0
+        out_shape = out_shape + left_shape[:-1] + \
+                    right_shape[0:right_critical_dim] + \
+                    right_shape[right_critical_dim + 1:]
+    elif len(right_shape) > 0:
+        #dot does not reduce for scalars
+        out_shape = out_shape + left_shape[:-1]
+    if my_critical_dim_size != right_critical_dim_size:
+        raise OperationError(space.w_ValueError, space.wrap(
+                                        "objects are not aligned"))
+    return out_shape, right_critical_dim
+
 class __extend__(W_NDimArray):
     @jit.unroll_safe
     def descr_get_shape(self, space):
@@ -180,8 +201,6 @@
             return self.implementation.descr_getitem(space, self, w_idx)
         except ArrayArgumentException:
             return self.getitem_array_int(space, w_idx)
-        except OperationError:
-            raise OperationError(space.w_IndexError, space.wrap("wrong index"))
 
     def getitem(self, space, index_list):
         return self.implementation.getitem_index(space, index_list)
@@ -199,6 +218,10 @@
         except ArrayArgumentException:
             self.setitem_array_int(space, w_idx, w_value)
 
+    def descr_delitem(self, space, w_idx):
+        raise OperationError(space.w_ValueError, space.wrap(
+            "cannot delete array elements"))
+
     def descr_len(self, space):
         shape = self.get_shape()
         if len(shape):
@@ -820,7 +843,7 @@
             # numpy compatability
             return W_NDimArray.new_scalar(space, dtype, space.wrap(0))
         # Do the dims match?
-        out_shape, other_critical_dim = match_dot_shapes(space, self, other)
+        out_shape, other_critical_dim = _match_dot_shapes(space, self, other)
         w_res = W_NDimArray.from_shape(space, out_shape, dtype, w_instance=self)
         # This is the place to add fpypy and blas
         return loop.multidim_dot(space, self, other,  w_res, dtype,
@@ -838,7 +861,7 @@
     # ----------------------- reduce -------------------------------
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False,
-                           cumultative=False):
+                           cumulative=False):
         def impl(self, space, w_axis=None, w_dtype=None, w_out=None):
             if space.is_none(w_out):
                 out = None
@@ -849,9 +872,9 @@
                 out = w_out
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(
                 space, self, promote_to_largest, w_axis,
-                False, out, w_dtype, cumultative=cumultative)
+                False, out, w_dtype, cumulative=cumulative)
         return func_with_new_name(impl, "reduce_%s_impl_%d_%d" % (ufunc_name,
-                    promote_to_largest, cumultative))
+                    promote_to_largest, cumulative))
 
     descr_sum = _reduce_ufunc_impl("add")
     descr_sum_promote = _reduce_ufunc_impl("add", True)
@@ -861,8 +884,8 @@
     descr_all = _reduce_ufunc_impl('logical_and')
     descr_any = _reduce_ufunc_impl('logical_or')
 
-    descr_cumsum = _reduce_ufunc_impl('add', cumultative=True)
-    descr_cumprod = _reduce_ufunc_impl('multiply', cumultative=True)
+    descr_cumsum = _reduce_ufunc_impl('add', cumulative=True)
+    descr_cumprod = _reduce_ufunc_impl('multiply', cumulative=True)
 
     def _reduce_argmax_argmin_impl(op_name):
         def impl(self, space, w_axis=None, w_out=None):
@@ -1046,6 +1069,7 @@
     __len__ = interp2app(W_NDimArray.descr_len),
     __getitem__ = interp2app(W_NDimArray.descr_getitem),
     __setitem__ = interp2app(W_NDimArray.descr_setitem),
+    __delitem__ = interp2app(W_NDimArray.descr_delitem),
 
     __repr__ = interp2app(W_NDimArray.descr_repr),
     __str__ = interp2app(W_NDimArray.descr_str),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -9,6 +9,7 @@
 from pypy.module.micronumpy.interp_support import unwrap_axis_arg
 from pypy.module.micronumpy.strides import shape_agreement
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
+from pypy.module.micronumpy.constants import *
 
 def done_if_true(dtype, val):
     return dtype.itemtype.bool(val)
@@ -86,7 +87,7 @@
             out = w_out
         return self.reduce(space, w_obj, False, #do not promote_to_largest
                     w_axis, True, #keepdims must be true
-                    out, w_dtype, cumultative=True)
+                    out, w_dtype, cumulative=True)
 
     @unwrap_spec(skipna=bool, keepdims=bool)
     def descr_reduce(self, space, w_obj, w_axis=None, w_dtype=None,
@@ -158,7 +159,7 @@
                            w_dtype)
 
     def reduce(self, space, w_obj, promote_to_largest, w_axis,
-               keepdims=False, out=None, dtype=None, cumultative=False):
+               keepdims=False, out=None, dtype=None, cumulative=False):
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
                 "supported for binary functions"))
@@ -192,7 +193,7 @@
                                 "%s.reduce without identity", self.name)
         if shapelen > 1 and axis < shapelen:
             temp = None
-            if cumultative:
+            if cumulative:
                 shape = obj_shape[:]
                 temp_shape = obj_shape[:axis] + obj_shape[axis + 1:]
                 if out:
@@ -226,15 +227,15 @@
             else:
                 out = W_NDimArray.from_shape(space, shape, dtype, w_instance=obj)
             return loop.do_axis_reduce(shape, self.func, obj, dtype, axis, out,
-                                       self.identity, cumultative, temp)
-        if cumultative:
+                                       self.identity, cumulative, temp)
+        if cumulative:
             if out:
                 if out.get_shape() != [obj.get_size()]:
                     raise OperationError(space.w_ValueError, space.wrap(
                         "out of incompatible size"))
             else:
                 out = W_NDimArray.from_shape(space, [obj.get_size()], dtype, w_instance=obj)
-            loop.compute_reduce_cumultative(obj, out, dtype, self.func,
+            loop.compute_reduce_cumulative(obj, out, dtype, self.func,
                                             self.identity)
             return out
         if out:
@@ -431,16 +432,16 @@
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
     # Some operations promote op(bool, bool) to return int8, rather than bool
-    if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
+    if promote_bools and (dt1.kind == dt2.kind == NPY_GENBOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
 
     # Everything numeric promotes to complex
     if dt2.is_complex_type() or dt1.is_complex_type():
-        if dt2.num == 14:
+        if dt2.num == NPY_CFLOAT:
             return interp_dtype.get_dtype_cache(space).w_complex64dtype
-        elif dt2.num == 15:
+        elif dt2.num == NPY_CDOUBLE:
             return interp_dtype.get_dtype_cache(space).w_complex128dtype
-        elif dt2.num == 16:
+        elif dt2.num == NPY_CLONGDOUBLE:
             return interp_dtype.get_dtype_cache(space).w_complexlongdtype
         else:
             raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
@@ -452,35 +453,30 @@
         return dt2
 
     # Everything promotes to float, and bool promotes to everything.
-    if dt2.kind == interp_dtype.FLOATINGLTR or dt1.kind == interp_dtype.BOOLLTR:
+    if dt2.kind == NPY_FLOATINGLTR or dt1.kind == NPY_GENBOOLLTR:
         # Float32 + 8-bit int = Float64
-        if dt2.num == 11 and dt1.itemtype.get_element_size() >= 4:
+        if dt2.num == NPY_FLOAT and dt1.itemtype.get_element_size() >= 4:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
         return dt2
 
     # for now this means mixing signed and unsigned
-    if dt2.kind == interp_dtype.SIGNEDLTR:
+    if dt2.kind == NPY_SIGNEDLTR:
         # if dt2 has a greater number of bytes, then just go with it
         if dt1.itemtype.get_element_size() < dt2.itemtype.get_element_size():
             return dt2
         # we need to promote both dtypes
         dtypenum = dt2.num + 2
-    elif dt2.num == 10 or (LONG_BIT == 64 and dt2.num == 8):
+    elif dt2.num == NPY_ULONGLONG or (LONG_BIT == 64 and dt2.num == NPY_ULONG):
         # UInt64 + signed = Float64
-        dtypenum = 12
+        dtypenum = NPY_DOUBLE
     elif dt2.is_flexible_type():
         # For those operations that get here (concatenate, stack),
         # flexible types take precedence over numeric type
         if dt2.is_record_type():
             return dt2
         if dt1.is_str_or_unicode():
-            if dt2.num == 18:
-                if dt2.itemtype.get_element_size() >= \
-                           dt1.itemtype.get_element_size():
-                    return dt2
-                return dt1
             if dt2.itemtype.get_element_size() >= \
-                       dt1.itemtype.get_element_size():
+                    dt1.itemtype.get_element_size():
                 return dt2
             return dt1
         return dt2
@@ -490,7 +486,7 @@
     newdtype = interp_dtype.get_dtype_cache(space).dtypes_by_num[dtypenum]
 
     if (newdtype.itemtype.get_element_size() > dt2.itemtype.get_element_size() or
-        newdtype.kind == interp_dtype.FLOATINGLTR):
+        newdtype.kind == NPY_FLOATINGLTR):
         return newdtype
     else:
         # we only promoted to long on 32-bit or to longlong on 64-bit
@@ -501,23 +497,23 @@
 @jit.unroll_safe
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
         promote_bools=False, promote_to_largest=False):
-    if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
+    if promote_bools and (dt.kind == NPY_GENBOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
     if promote_to_float:
-        if dt.kind == interp_dtype.FLOATINGLTR or dt.kind==interp_dtype.COMPLEXLTR:
+        if dt.kind == NPY_FLOATINGLTR or dt.kind == NPY_COMPLEXLTR:
             return dt
-        if dt.num >= 5:
+        if dt.num >= NPY_INT:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
         for bytes, dtype in interp_dtype.get_dtype_cache(space).float_dtypes_by_num_bytes:
-            if (dtype.kind == interp_dtype.FLOATINGLTR and
+            if (dtype.kind == NPY_FLOATINGLTR and
                 dtype.itemtype.get_element_size() > dt.itemtype.get_element_size()):
                 return dtype
     if promote_to_largest:
-        if dt.kind == interp_dtype.BOOLLTR or dt.kind == interp_dtype.SIGNEDLTR:
+        if dt.kind == NPY_GENBOOLLTR or dt.kind == NPY_SIGNEDLTR:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
-        elif dt.kind == interp_dtype.FLOATINGLTR:
+        elif dt.kind == NPY_FLOATINGLTR:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
-        elif dt.kind == interp_dtype.UNSIGNEDLTR:
+        elif dt.kind == NPY_UNSIGNEDLTR:
             return interp_dtype.get_dtype_cache(space).w_uint64dtype
         else:
             assert False
@@ -559,8 +555,8 @@
         if (current_guess is None):
             return interp_dtype.variable_dtype(space,
                                                'S%d' % space.len_w(w_obj))
-        elif current_guess.num ==18:
-            if  current_guess.itemtype.get_size() < space.len_w(w_obj):
+        elif current_guess.num == NPY_STRING:
+            if current_guess.itemtype.get_size() < space.len_w(w_obj):
                 return interp_dtype.variable_dtype(space,
                                                    'S%d' % space.len_w(w_obj))
         return current_guess
diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py
--- a/pypy/module/micronumpy/iter.py
+++ b/pypy/module/micronumpy/iter.py
@@ -275,11 +275,11 @@
         return self.indexes[d]
 
 class AxisIterator(base.BaseArrayIterator):
-    def __init__(self, array, shape, dim, cumultative):
+    def __init__(self, array, shape, dim, cumulative):
         self.shape = shape
         strides = array.get_strides()
         backstrides = array.get_backstrides()
-        if cumultative:
+        if cumulative:
             self.strides = strides
             self.backstrides = backstrides
         elif len(shape) == len(strides):
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -10,8 +10,8 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.iter import PureShapeIterator
-from pypy.module.micronumpy import constants
-from pypy.module.micronumpy.support import int_w
+from pypy.module.micronumpy.support import index_w
+from pypy.module.micronumpy.constants import *
 
 call2_driver = jit.JitDriver(name='numpy_call2',
                              greens = ['shapelen', 'func', 'calc_dtype',
@@ -159,10 +159,16 @@
                                   greens = ['shapelen', 'func', 'dtype'],
                                   reds = 'auto')
 
-def compute_reduce_cumultative(obj, out, calc_dtype, func, identity):
+def compute_reduce_cumulative(obj, out, calc_dtype, func, identity):
     obj_iter = obj.create_iter()
     out_iter = out.create_iter()
-    cur_value = identity.convert_to(calc_dtype)
+    if identity is None:
+        cur_value = obj_iter.getitem().convert_to(calc_dtype)
+        out_iter.setitem(cur_value)
+        out_iter.next()
+        obj_iter.next()
+    else:
+        cur_value = identity.convert_to(calc_dtype)
     shapelen = len(obj.get_shape())
     while not obj_iter.done():
         reduce_cum_driver.jit_merge_point(shapelen=shapelen, func=func,
@@ -218,10 +224,10 @@
                                             'func', 'dtype'],
                                     reds='auto')
 
-def do_axis_reduce(shape, func, arr, dtype, axis, out, identity, cumultative,
+def do_axis_reduce(shape, func, arr, dtype, axis, out, identity, cumulative,
                    temp):
-    out_iter = out.create_axis_iter(arr.get_shape(), axis, cumultative)
-    if cumultative:
+    out_iter = out.create_axis_iter(arr.get_shape(), axis, cumulative)
+    if cumulative:
         temp_iter = temp.create_axis_iter(arr.get_shape(), axis, False)
     else:
         temp_iter = out_iter # hack
@@ -240,7 +246,7 @@
             cur = temp_iter.getitem()
             w_val = func(dtype, cur, w_val)
         out_iter.setitem(w_val)
-        if cumultative:
+        if cumulative:
             temp_iter.setitem(w_val)
             temp_iter.next()
         arr_iter.next()
@@ -581,15 +587,15 @@
     while not arr_iter.done():
         choose_driver.jit_merge_point(shapelen=shapelen, dtype=dtype,
                                       mode=mode)
-        index = int_w(space, arr_iter.getitem())
+        index = index_w(space, arr_iter.getitem())
         if index < 0 or index >= len(iterators):
-            if mode == constants.MODE_RAISE:
+            if mode == NPY_RAISE:
                 raise OperationError(space.w_ValueError, space.wrap(
                     "invalid entry in choice array"))
-            elif mode == constants.MODE_WRAP:
+            elif mode == NPY_WRAP:
                 index = index % (len(iterators))
             else:
-                assert mode == constants.MODE_CLIP
+                assert mode == NPY_CLIP
                 if index < 0:
                     index = 0
                 else:
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,11 +1,15 @@
 from rpython.rlib import jit
 from pypy.interpreter.error import OperationError
 
-def int_w(space, w_obj):
+def index_w(space, w_obj):
     try:
         return space.int_w(space.index(w_obj))
     except OperationError:
-        return space.int_w(space.int(w_obj))
+        try:
+            return space.int_w(space.int(w_obj))
+        except OperationError:
+            raise OperationError(space.w_IndexError, space.wrap(
+                "cannot convert index to integer"))
 
 @jit.unroll_safe
 def product(s):
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,4 +1,4 @@
-from pypy.module.micronumpy.interp_dtype import NATBYTE, OPPBYTE
+from pypy.module.micronumpy.interp_dtype import NPY_NATBYTE, NPY_OPPBYTE
 from pypy.conftest import option
 
 class BaseNumpyAppTest(object):
@@ -22,5 +22,5 @@
             import sys
             sys.modules['numpypy'] = numpy
             """)
-        cls.w_non_native_prefix = cls.space.wrap(OPPBYTE)
-        cls.w_native_prefix = cls.space.wrap(NATBYTE)
+        cls.w_non_native_prefix = cls.space.wrap(NPY_OPPBYTE)
+        cls.w_native_prefix = cls.space.wrap(NPY_NATBYTE)
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -34,6 +34,14 @@
 
         assert dtype(None) is dtype(float)
 
+        e = dtype('int8')
+        exc = raises(KeyError, "e[2]")
+        assert exc.value.message == "There are no fields in dtype int8."
+        exc = raises(KeyError, "e['z']")
+        assert exc.value.message == "There are no fields in dtype int8."
+        exc = raises(KeyError, "e[None]")
+        assert exc.value.message == "There are no fields in dtype int8."
+
         exc = raises(TypeError, dtype, (1, 2))
         assert 'data type not understood' in str(exc.value)
         raises(KeyError, 'dtype(int)["asdasd"]')
@@ -828,7 +836,17 @@
         assert d["x"].itemsize == 16
         e = dtype([("x", "float", 2), ("y", "int", 2)])
         assert e.fields.keys() == keys
-        assert e['x'].shape == (2,)
+        for v in ['x', u'x', 0, -2]:
+            assert e[v] == (dtype('float'), (2,))
+        for v in ['y', u'y', 1, -1]:
+            assert e[v] == (dtype('int'), (2,))
+        for v in [-3, 2]:
+            exc = raises(IndexError, "e[%d]" % v)
+            assert exc.value.message == "Field index %d out of range." % v
+        exc = raises(KeyError, "e['z']")
+        assert exc.value.message == "Field named 'z' not found."
+        exc = raises(ValueError, "e[None]")
+        assert exc.value.message == 'Field key must be an integer, string, or unicode.'
 
         dt = dtype((float, 10))
         assert dt.shape == (10,)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -511,6 +511,12 @@
         a[self.CustomIntObject(1)] = 100
         assert a[1] == 100
 
+    def test_delitem(self):
+        import numpypy as np
+        a = np.arange(10)
+        exc = raises(ValueError, 'del a[2]')
+        assert exc.value.message == 'cannot delete array elements'
+
     def test_access_swallow_exception(self):
         class ErrorIndex(object):
             def __index__(self):
@@ -525,8 +531,10 @@
 
         from numpypy import arange
         a = arange(10)
-        raises(IndexError, "a[ErrorIndex()] == 0")
-        raises(IndexError, "a[ErrorInt()] == 0")
+        exc = raises(IndexError, "a[ErrorIndex()] == 0")
+        assert exc.value.message == 'cannot convert index to integer'
+        exc = raises(IndexError, "a[ErrorInt()] == 0")
+        assert exc.value.message == 'cannot convert index to integer'
 
     def test_setslice_array(self):
         from numpypy import array
@@ -996,6 +1004,13 @@
         b = a // 2
         assert (b == [0, 0, 1, 1, 2]).all()
 
+    def test_signed_integer_division_overflow(self):
+        import numpypy as np
+        for s in (8, 16, 32, 64):
+            for o in ['__div__', '__floordiv__']:
+                a = np.array([-2**(s-1)], dtype='int%d' % s)
+                assert getattr(a, o)(-1) == 0
+
     def test_truediv(self):
         from operator import truediv
         from numpypy import arange
@@ -1778,6 +1793,15 @@
         raises(IndexError, "arange(10)[array([10])] = 3")
         raises(IndexError, "arange(10)[[-11]] = 3")
 
+    def test_bool_single_index(self):
+        import numpypy as np
+        a = np.array([[1, 2, 3],
+                      [4, 5, 6],
+                      [7, 8, 9]])
+        a[np.array(True)]; skip("broken")  # check for crash but skip rest of test until correct
+        assert (a[np.array(True)] == a[1]).all()
+        assert (a[np.array(False)] == a[0]).all()
+
     def test_bool_array_index(self):
         from numpypy import arange, array
         b = arange(10)
@@ -2905,10 +2929,22 @@
         d = dtype([("x", "int", 3), ("y", "float", 5)])
         a = array([([1, 2, 3], [0.5, 1.5, 2.5, 3.5, 4.5]), ([4, 5, 6], [5.5, 6.5, 7.5, 8.5, 9.5])], dtype=d)
 
-        assert (a[0]["x"] == [1, 2, 3]).all()
-        assert (a[0]["y"] == [0.5, 1.5, 2.5, 3.5, 4.5]).all()
-        assert (a[1]["x"] == [4, 5, 6]).all()
-        assert (a[1]["y"] == [5.5, 6.5, 7.5, 8.5, 9.5]).all()
+        for v in ['x', u'x', 0, -2]:
+            assert (a[0][v] == [1, 2, 3]).all()
+            assert (a[1][v] == [4, 5, 6]).all()
+        for v in ['y', u'y', -1, 1]:
+            assert (a[0][v] == [0.5, 1.5, 2.5, 3.5, 4.5]).all()
+            assert (a[1][v] == [5.5, 6.5, 7.5, 8.5, 9.5]).all()
+        for v in [-3, 2]:
+            exc = raises(IndexError, "a[0][%d]" % v)
+            assert exc.value.message == "invalid index (%d)" % (v + 2 if v < 0 else v)
+        exc = raises(IndexError, "a[0]['z']")
+        assert exc.value.message == "invalid index"
+        exc = raises(IndexError, "a[0][None]")
+        assert exc.value.message == "invalid index"
+
+        exc = raises(IndexError, "a[0][None]")
+        assert exc.value.message == 'invalid index'
 
         a[0]["x"][0] = 200
         assert a[0]["x"][0] == 200
@@ -2929,6 +2965,8 @@
         a[0, 0] = 500
         assert (a[0, 0, 0] == 500).all()
         assert a[0, 0, 0].shape == (10,)
+        exc = raises(ValueError, "a[0, 0]['z']")
+        assert exc.value.message == 'field named z not found'
 
     def test_subarray_multiple_rows(self):
         import numpypy as np
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1073,3 +1073,14 @@
         print b
         assert (b == [[0, 0, 1], [1, 3, 5]]).all()
         assert b.dtype == int
+
+    def test_noncommutative_reduce_accumulate(self):
+        import numpypy as np
+        tosubtract = np.arange(5)
+        todivide = np.array([2.0, 0.5, 0.25])
+        assert np.subtract.reduce(tosubtract) == -10
+        assert np.divide.reduce(todivide) == 16.0
+        assert (np.subtract.accumulate(tosubtract) ==
+                np.array([0, -1, -3, -6, -10])).all()
+        assert (np.divide.accumulate(todivide) ==
+                np.array([2., 4., 16.])).all()
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -115,6 +115,10 @@
 class BaseType(object):
     SortRepr = None # placeholders for sorting classes, overloaded in sort.py
     Sort = None
+    _immutable_fields_ = ['native']
+
+    def __init__(self, native=True):
+        self.native = native
 
     def _unimplemented_ufunc(self, *args):
         raise NotImplementedError
@@ -172,7 +176,15 @@
         raise NotImplementedError
 
     def _read(self, storage, i, offset):
-        return raw_storage_getitem(self.T, storage, i + offset)
+        res = raw_storage_getitem(self.T, storage, i + offset)
+        if not self.native:
+            res = byteswap(res)
+        return res
+
+    def _write(self, storage, i, offset, value):
+        if not self.native:
+            value = byteswap(value)
+        raw_storage_setitem(storage, i + offset, value)
 
     def read(self, arr, i, offset, dtype=None):
         return self.box(self._read(arr.storage, i, offset))
@@ -180,9 +192,6 @@
     def read_bool(self, arr, i, offset):
         return bool(self.for_computation(self._read(arr.storage, i, offset)))
 
-    def _write(self, storage, i, offset, value):
-        raw_storage_setitem(storage, i + offset, value)
-
     def store(self, arr, i, offset, box):
         self._write(arr.storage, i, offset, self.unbox(box))
 
@@ -307,17 +316,6 @@
         float64 = Float64()
         return float64.rint(float64.box(v))
 
-class NonNativePrimitive(Primitive):
-    _mixin_ = True
-
-    def _read(self, storage, i, offset):
-        res = raw_storage_getitem(self.T, storage, i + offset)
-        return byteswap(res)
-
-    def _write(self, storage, i, offset, value):
-        value = byteswap(value)
-        raw_storage_setitem(storage, i + offset, value)
-
 class Bool(BaseType, Primitive):
     T = lltype.Bool
     BoxType = interp_boxes.W_BoolBox
@@ -402,8 +400,6 @@
             return 1
         return 0
 
-NonNativeBool = Bool
-
 class Integer(Primitive):
     _mixin_ = True
 
@@ -422,17 +418,29 @@
     def default_fromstring(self, space):
         return self.box(0)
 
-    @simple_binary_op
-    def div(self, v1, v2):
+    @specialize.argtype(1, 2)
+    def div(self, b1, b2):
+        v1 = self.for_computation(self.unbox(b1))
+        v2 = self.for_computation(self.unbox(b2))
         if v2 == 0:
-            return 0
-        return v1 / v2
+            return self.box(0)
+        if (self.T is rffi.SIGNEDCHAR or self.T is rffi.SHORT or self.T is rffi.INT or
+                self.T is rffi.LONG or self.T is rffi.LONGLONG):
+            if v2 == -1 and v1 == self.for_computation(most_neg_value_of(self.T)):
+                return self.box(0)
+        return self.box(v1 / v2)
 
-    @simple_binary_op
-    def floordiv(self, v1, v2):
+    @specialize.argtype(1, 2)
+    def floordiv(self, b1, b2):
+        v1 = self.for_computation(self.unbox(b1))
+        v2 = self.for_computation(self.unbox(b2))
         if v2 == 0:
-            return 0
-        return v1 // v2
+            return self.box(0)
+        if (self.T is rffi.SIGNEDCHAR or self.T is rffi.SHORT or self.T is rffi.INT or
+                self.T is rffi.LONG or self.T is rffi.LONGLONG):
+            if v2 == -1 and v1 == self.for_computation(most_neg_value_of(self.T)):
+                return self.box(0)
+        return self.box(v1 // v2)
 
     @simple_binary_op
     def mod(self, v1, v2):
@@ -531,83 +539,46 @@
     def signbit(self, v):
         return v < 0
 
-class NonNativeInteger(NonNativePrimitive, Integer):
-    _mixin_ = True
-
 class Int8(BaseType, Integer):
     T = rffi.SIGNEDCHAR
     BoxType = interp_boxes.W_Int8Box
     format_code = "b"
 
-NonNativeInt8 = Int8
-
 class UInt8(BaseType, Integer):
     T = rffi.UCHAR
     BoxType = interp_boxes.W_UInt8Box
     format_code = "B"
 
-NonNativeUInt8 = UInt8
-
 class Int16(BaseType, Integer):
     T = rffi.SHORT
     BoxType = interp_boxes.W_Int16Box
     format_code = "h"
 
-class NonNativeInt16(BaseType, NonNativeInteger):
-    T = rffi.SHORT
-    BoxType = interp_boxes.W_Int16Box
-    format_code = "h"
-
 class UInt16(BaseType, Integer):
     T = rffi.USHORT
     BoxType = interp_boxes.W_UInt16Box
     format_code = "H"
 
-class NonNativeUInt16(BaseType, NonNativeInteger):
-    T = rffi.USHORT
-    BoxType = interp_boxes.W_UInt16Box
-    format_code = "H"
-
 class Int32(BaseType, Integer):
     T = rffi.INT
     BoxType = interp_boxes.W_Int32Box
     format_code = "i"
 
-class NonNativeInt32(BaseType, NonNativeInteger):
-    T = rffi.INT
-    BoxType = interp_boxes.W_Int32Box
-    format_code = "i"
-
 class UInt32(BaseType, Integer):
     T = rffi.UINT
     BoxType = interp_boxes.W_UInt32Box
     format_code = "I"
 
-class NonNativeUInt32(BaseType, NonNativeInteger):
-    T = rffi.UINT
-    BoxType = interp_boxes.W_UInt32Box
-    format_code = "I"
-
 class Long(BaseType, Integer):
     T = rffi.LONG
     BoxType = interp_boxes.W_LongBox
     format_code = "l"
 
-class NonNativeLong(BaseType, NonNativeInteger):
-    T = rffi.LONG
-    BoxType = interp_boxes.W_LongBox
-    format_code = "l"
-
 class ULong(BaseType, Integer):
     T = rffi.ULONG
     BoxType = interp_boxes.W_ULongBox
     format_code = "L"
 
-class NonNativeULong(BaseType, NonNativeInteger):
-    T = rffi.ULONG
-    BoxType = interp_boxes.W_ULongBox
-    format_code = "L"
-
 def _int64_coerce(self, space, w_item):
     try:
         return self._base_coerce(space, w_item)
@@ -628,13 +599,6 @@
 
     _coerce = func_with_new_name(_int64_coerce, '_coerce')
 
-class NonNativeInt64(BaseType, NonNativeInteger):
-    T = rffi.LONGLONG
-    BoxType = interp_boxes.W_Int64Box
-    format_code = "q"
-
-    _coerce = func_with_new_name(_int64_coerce, '_coerce')
-
 def _uint64_coerce(self, space, w_item):
     try:
         return self._base_coerce(space, w_item)
@@ -655,13 +619,6 @@
 
     _coerce = func_with_new_name(_uint64_coerce, '_coerce')
 
-class NonNativeUInt64(BaseType, NonNativeInteger):
-    T = rffi.ULONGLONG
-    BoxType = interp_boxes.W_UInt64Box
-    format_code = "Q"
-
-    _coerce = func_with_new_name(_uint64_coerce, '_coerce')
-
 class Float(Primitive):
     _mixin_ = True
 
@@ -986,20 +943,7 @@
         else:
             return x
 
-class NonNativeFloat(NonNativePrimitive, Float):
-    _mixin_ = True
-
-    def _read(self, storage, i, offset):
-        res = raw_storage_getitem(self.T, storage, i + offset)
-        return byteswap(res)
-
-    def _write(self, storage, i, offset, value):
-        swapped_value = byteswap(rffi.cast(self.T, value))
-        raw_storage_setitem(storage, i + offset, swapped_value)
-
-class BaseFloat16(Float):
-    _mixin_ = True
-
+class Float16(BaseType, Float):
     _STORAGE_T = rffi.USHORT
     T = rffi.SHORT
     BoxType = interp_boxes.W_Float16Box
@@ -1022,46 +966,29 @@
         swapped = byteswap(rffi.cast(self._STORAGE_T, hbits))
         return self.box(float_unpack(r_ulonglong(swapped), 2))
 
-class Float16(BaseType, BaseFloat16):
     def _read(self, storage, i, offset):
         hbits = raw_storage_getitem(self._STORAGE_T, storage, i + offset)
+        if not self.native:
+            hbits = byteswap(hbits)
         return float_unpack(r_ulonglong(hbits), 2)
 
     def _write(self, storage, i, offset, value):
-        hbits = float_pack(value,2)
+        hbits = rffi.cast(self._STORAGE_T, float_pack(value, 2))
+        if not self.native:
+            hbits = byteswap(hbits)
         raw_storage_setitem(storage, i + offset,
                 rffi.cast(self._STORAGE_T, hbits))
 
-class NonNativeFloat16(BaseType, BaseFloat16):
-    def _read(self, storage, i, offset):
-        hbits = raw_storage_getitem(self._STORAGE_T, storage, i + offset)
-        return float_unpack(r_ulonglong(byteswap(hbits)), 2)
-
-    def _write(self, storage, i, offset, value):
-        hbits = float_pack(value,2)
-        raw_storage_setitem(storage, i + offset,
-                byteswap(rffi.cast(self._STORAGE_T, hbits)))
-
 class Float32(BaseType, Float):
     T = rffi.FLOAT
     BoxType = interp_boxes.W_Float32Box
     format_code = "f"
 
-class NonNativeFloat32(BaseType, NonNativeFloat):
-    T = rffi.FLOAT
-    BoxType = interp_boxes.W_Float32Box
-    format_code = "f"
-
 class Float64(BaseType, Float):
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Float64Box
     format_code = "d"
 
-class NonNativeFloat64(BaseType, NonNativeFloat):
-    T = rffi.DOUBLE
-    BoxType = interp_boxes.W_Float64Box
-    format_code = "d"
-
 class ComplexFloating(object):
     _mixin_ = True
 
@@ -1613,21 +1540,14 @@
     BoxType = interp_boxes.W_Complex64Box
     ComponentBoxType = interp_boxes.W_Float32Box
 
-NonNativeComplex64 = Complex64
-
 class Complex128(ComplexFloating, BaseType):
     T = rffi.DOUBLE
     BoxType = interp_boxes.W_Complex128Box
     ComponentBoxType = interp_boxes.W_Float64Box
 
-NonNativeComplex128 = Complex128
-
 if interp_boxes.long_double_size == 8:
     FloatLong = Float64
-    NonNativeFloatLong = NonNativeFloat64
-
     ComplexLong = Complex128
-    NonNativeComplexLong = NonNativeComplex128
 
 elif interp_boxes.long_double_size in (12, 16):
     class FloatLong(BaseType, Float):
@@ -1645,19 +1565,16 @@
             pack_float80(result, value, 10, not native_is_bigendian)
             return self.box(unpack_float80(result.build(), native_is_bigendian))
 
-    NonNativeFloatLong = FloatLong
-
     class ComplexLong(ComplexFloating, BaseType):
         T = rffi.LONGDOUBLE
         BoxType = interp_boxes.W_ComplexLongBox
         ComponentBoxType = interp_boxes.W_FloatLongBox
 
-    NonNativeComplexLong = ComplexLong
-
-class BaseStringType(object):
-    _mixin_ = True
+class BaseStringType(BaseType):
+    _immutable_fields = ['size']
 
     def __init__(self, size=0):
+        BaseType.__init__(self)
         self.size = size
 
     def get_element_size(self):
@@ -1683,7 +1600,7 @@
         )
     return dispatcher
 
-class StringType(BaseType, BaseStringType):
+class StringType(BaseStringType):
     T = lltype.Char
 
     @jit.unroll_safe
@@ -1800,9 +1717,7 @@
         for i in xrange(start, stop, width):
             self._store(storage, i, offset, box)
 
-NonNativeStringType = StringType
-
-class UnicodeType(BaseType, BaseStringType):
+class UnicodeType(BaseStringType):
     T = lltype.UniChar
 
     @jit.unroll_safe
@@ -1812,9 +1727,7 @@
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "coerce (probably from set_item) not implemented for unicode type"))
 
-NonNativeUnicodeType = UnicodeType
-
-class VoidType(BaseType, BaseStringType):
+class VoidType(BaseStringType):
     T = lltype.Char
 
     def _coerce(self, space, arr, ofs, dtype, w_items, shape):
@@ -1858,12 +1771,12 @@
                              dtype.shape, arr, W_NDimArray(arr), dtype.subdtype)
         return W_NDimArray(implementation)
 
-NonNativeVoidType = VoidType
-
 class RecordType(BaseType):
     T = lltype.Char
+    _immutable_fields_ = ['offsets_and_fields', 'size']
 
     def __init__(self, offsets_and_fields, size):
+        BaseType.__init__(self)
         self.offsets_and_fields = offsets_and_fields
         self.size = size
 

From noreply at buildbot.pypy.org  Tue Oct 29 20:33:18 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 20:33:18 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131029193318.DCA751C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67705:ee3998d57f80
Date: 2013-10-29 15:32 -0400
http://bitbucket.org/pypy/pypy/changeset/ee3998d57f80/

Log:	merge

diff too long, truncating to 2000 out of 2944 lines

diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -6,7 +6,8 @@
 import operator
 from rpython.tool.pairtype import pair, pairtype
 from rpython.annotator.model import SomeObject, SomeInteger, SomeBool, s_Bool
-from rpython.annotator.model import SomeString, SomeChar, SomeList, SomeDict
+from rpython.annotator.model import SomeString, SomeChar, SomeList, SomeDict,\
+     SomeOrderedDict
 from rpython.annotator.model import SomeUnicodeCodePoint, SomeUnicodeString
 from rpython.annotator.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue
 from rpython.annotator.model import SomeInstance, SomeBuiltin, SomeIterator
@@ -581,7 +582,8 @@
 class __extend__(pairtype(SomeDict, SomeDict)):
 
     def union((dic1, dic2)):
-        return SomeDict(dic1.dictdef.union(dic2.dictdef))
+        assert dic1.__class__ == dic2.__class__
+        return dic1.__class__(dic1.dictdef.union(dic2.dictdef))
 
 
 class __extend__(pairtype(SomeDict, SomeObject)):
@@ -840,6 +842,7 @@
 _make_none_union('SomeString',      'no_nul=obj.no_nul, can_be_None=True')
 _make_none_union('SomeUnicodeString', 'can_be_None=True')
 _make_none_union('SomeList',         'obj.listdef')
+_make_none_union('SomeOrderedDict',          'obj.dictdef')
 _make_none_union('SomeDict',          'obj.dictdef')
 _make_none_union('SomeWeakRef',         'obj.classdef')
 
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -5,9 +5,10 @@
 from __future__ import absolute_import
 
 import sys, types, inspect, weakref
+from collections import OrderedDict
 
 from rpython.flowspace.model import Constant
-from rpython.annotator.model import (
+from rpython.annotator.model import (SomeOrderedDict,
     SomeString, SomeChar, SomeFloat, SomePtr, unionof, SomeInstance, SomeDict,
     SomeBuiltin, SomePBC, SomeInteger, TLS, SomeAddress, SomeUnicodeCodePoint,
     s_None, s_ImpossibleValue, SomeLLADTMeth, SomeBool, SomeTuple,
@@ -370,7 +371,7 @@
                 for e in x:
                     listdef.generalize(self.immutablevalue(e, False))
                 result = SomeList(listdef)
-        elif tp is dict or tp is r_dict:
+        elif tp is dict or tp is r_dict or tp is OrderedDict:
             if need_const:
                 key = Constant(x)
                 try:
@@ -412,7 +413,10 @@
                     dictdef.generalize_key(self.immutablevalue(ek, False))
                     dictdef.generalize_value(self.immutablevalue(ev, False))
                     dictdef.seen_prebuilt_key(ek)
-                result = SomeDict(dictdef)
+                if tp is OrderedDict:
+                    result = SomeOrderedDict(dictdef)
+                else:
+                    result = SomeDict(dictdef)
         elif tp is weakref.ReferenceType:
             x1 = x()
             if x1 is None:
diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -2,11 +2,13 @@
 Built-in functions.
 """
 import sys
+from collections import OrderedDict
 
 from rpython.annotator.model import (
     SomeInteger, SomeObject, SomeChar, SomeBool, SomeString, SomeTuple, s_Bool,
     SomeUnicodeCodePoint, SomeAddress, SomeFloat, unionof, SomeUnicodeString,
     SomePBC, SomeInstance, SomeDict, SomeList, SomeWeakRef, SomeIterator,
+    SomeOrderedDict,
     SomeByteArray, annotation_to_lltype, lltype_to_annotation,
     ll_to_annotation, add_knowntypedata, s_ImpossibleValue,)
 from rpython.annotator.bookkeeper import getbookkeeper
@@ -298,6 +300,10 @@
     dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn)
     return SomeDict(dictdef)
 
+def robjmodel_r_ordereddict(s_eqfn, s_hashfn):
+    dictdef = getbookkeeper().getdictdef(is_r_dict=True)
+    dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn)
+    return SomeOrderedDict(dictdef)
 
 def robjmodel_hlinvoke(s_repr, s_llcallable, *args_s):
     from rpython.rtyper import rmodel
@@ -357,6 +363,8 @@
 BUILTIN_ANALYZERS[rpython.rlib.rarithmetic.longlongmask] = rarith_longlongmask
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.instantiate] = robjmodel_instantiate
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_dict] = robjmodel_r_dict
+BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_ordereddict] = robjmodel_r_ordereddict
+BUILTIN_ANALYZERS[OrderedDict] = lambda : SomeOrderedDict(getbookkeeper().getdictdef())
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.hlinvoke] = robjmodel_hlinvoke
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.keepalive_until_here] = robjmodel_keepalive_until_here
 BUILTIN_ANALYZERS[rpython.rtyper.lltypesystem.llmemory.cast_ptr_to_adr] = llmemory_cast_ptr_to_adr
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -32,6 +32,7 @@
 import inspect
 import weakref
 from types import BuiltinFunctionType, MethodType
+from collections import OrderedDict
 
 import rpython
 from rpython.tool import descriptor
@@ -355,6 +356,18 @@
         else:
             return '{...%s...}' % (len(const),)
 
+class SomeOrderedDict(SomeDict):
+    knowntype = OrderedDict
+
+    def method_copy(dct):
+        return SomeOrderedDict(dct.dictdef)
+
+    def method_update(dct1, dct2):
+        if s_None.contains(dct2):
+            return SomeImpossibleValue()
+        assert isinstance(dct2, SomeOrderedDict), "OrderedDict.update(dict) not allowed"
+        dct1.dictdef.union(dct2.dictdef)
+
 
 class SomeIterator(SomeObject):
     "Stands for an iterator returning objects from a given container."
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -628,8 +628,11 @@
     The functions key_eq() and key_hash() are used by the key comparison
     algorithm."""
 
+    def _newdict(self):
+        return {}
+
     def __init__(self, key_eq, key_hash, force_non_null=False):
-        self._dict = {}
+        self._dict = self._newdict()
         self.key_eq = key_eq
         self.key_hash = key_hash
         self.force_non_null = force_non_null
@@ -664,7 +667,7 @@
         return dk.key, value
 
     def copy(self):
-        result = r_dict(self.key_eq, self.key_hash)
+        result = self.__class__(self.key_eq, self.key_hash)
         result.update(self)
         return result
 
@@ -700,6 +703,11 @@
     def __hash__(self):
         raise TypeError("cannot hash r_dict instances")
 
+class r_ordereddict(r_dict):
+    def _newdict(self):
+        from collections import OrderedDict
+        
+        return OrderedDict()
 
 class _r_dictkey(object):
     __slots__ = ['dic', 'key', 'hash']
@@ -735,7 +743,7 @@
     Function and staticmethod objects are duplicated, which means
     that annotation will not consider them as identical to another
     copy in another unrelated class.
-    
+
     By default, "special" methods and class attributes, with a name
     like "__xxx__", are not copied unless they are "__init__" or
     "__del__".  The list can be changed with the optional second
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -522,8 +522,10 @@
     A = lltype.typeOf(source)
     assert A == lltype.typeOf(dest)
     if isinstance(A.TO, lltype.GcArray):
-        assert isinstance(A.TO.OF, lltype.Ptr)
-        assert A.TO.OF.TO._gckind == 'gc'
+        if isinstance(A.TO.OF, lltype.Ptr):
+            assert A.TO.OF.TO._gckind == 'gc'
+        else:
+            assert isinstance(A.TO.OF, lltype.Struct)
     else:
         assert isinstance(A.TO, lltype.GcStruct)
         assert A.TO._arrayfld is not None
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -0,0 +1,1149 @@
+import sys
+from rpython.tool.pairtype import pairtype
+from rpython.flowspace.model import Constant
+from rpython.rtyper.rdict import AbstractDictRepr, AbstractDictIteratorRepr
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rlib import objectmodel, jit, rgc
+from rpython.rlib.debug import ll_assert
+from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rtyper import rmodel
+from rpython.rtyper.error import TyperError
+from rpython.rtyper.annlowlevel import llhelper
+
+
+# ____________________________________________________________
+#
+#  generic implementation of RPython dictionary, with parametric DICTKEY and
+#  DICTVALUE types. The basic implementation is a sparse array of indexes
+#  plus a dense array of structs that contain keys and values. struct looks
+#  like that:
+#
+#
+#    struct dictentry {
+#        DICTKEY key;
+#        DICTVALUE value;
+#        long f_hash;        # (optional) key hash, if hard to recompute
+#        bool f_valid;      # (optional) the entry is filled
+#    }
+#
+#    struct dicttable {
+#        int num_items;
+#        int num_used_items;
+#        int resize_counter;
+#        {byte, short, int, long} *indexes;
+#        dictentry *entries;
+#        lookup_function_no; # one of the four possible functions for different
+#                         # size dicts
+#        (Function DICTKEY, DICTKEY -> bool) *fnkeyeq;
+#        (Function DICTKEY -> int) *fnkeyhash;
+#    }
+#
+#
+
+def ll_call_lookup_function(d, key, hash, flag):
+    DICT = lltype.typeOf(d).TO
+    fun = d.lookup_function_no
+    if fun == FUNC_BYTE:
+        return DICT.lookup_family.byte_lookup_function(d, key, hash, flag)
+    elif fun == FUNC_SHORT:
+        return DICT.lookup_family.short_lookup_function(d, key, hash, flag)
+    elif IS_64BIT and fun == FUNC_INT:
+        return DICT.lookup_family.int_lookup_function(d, key, hash, flag)
+    elif fun == FUNC_LONG:
+        return DICT.lookup_family.long_lookup_function(d, key, hash, flag)
+    assert False
+
+def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
+                ll_fasthash_function=None, ll_hash_function=None,
+                ll_eq_function=None, method_cache={},
+                dummykeyobj=None, dummyvalueobj=None, rtyper=None,
+                setup_lookup_funcs=True):
+    # get the actual DICT type. if DICT is None, it's created, otherwise
+    # forward reference is becoming DICT
+    if DICT is None:
+        DICT = lltype.GcForwardReference()
+    # compute the shape of the DICTENTRY structure
+    entryfields = []
+    entrymeths = {
+        'allocate': lltype.typeMethod(_ll_malloc_entries),
+        'delete': _ll_free_entries,
+        'must_clear_key':   (isinstance(DICTKEY, lltype.Ptr)
+                             and DICTKEY._needsgc()),
+        'must_clear_value': (isinstance(DICTVALUE, lltype.Ptr)
+                             and DICTVALUE._needsgc()),
+        }
+
+    # * the key
+    entryfields.append(("key", DICTKEY))
+
+    # * the state of the entry - trying to encode it as dummy objects
+    if dummykeyobj:
+        # all the state can be encoded in the key
+        entrymeths['dummy_obj'] = dummykeyobj
+        entrymeths['valid'] = ll_valid_from_key
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_key
+        # the key is overwritten by 'dummy' when the entry is deleted
+        entrymeths['must_clear_key'] = False
+
+    elif dummyvalueobj:
+        # all the state can be encoded in the value
+        entrymeths['dummy_obj'] = dummyvalueobj
+        entrymeths['valid'] = ll_valid_from_value
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_value
+        # value is overwritten by 'dummy' when entry is deleted
+        entrymeths['must_clear_value'] = False
+
+    else:
+        # we need a flag to know if the entry was ever used
+        entryfields.append(("f_valid", lltype.Bool))
+        entrymeths['valid'] = ll_valid_from_flag
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_flag
+
+    # * the value
+    entryfields.append(("value", DICTVALUE))
+
+    if ll_fasthash_function is None:
+        entryfields.append(("f_hash", lltype.Signed))
+        entrymeths['hash'] = ll_hash_from_cache
+    else:
+        entrymeths['hash'] = ll_hash_recomputed
+        entrymeths['fasthashfn'] = ll_fasthash_function
+
+    # Build the lltype data structures
+    DICTENTRY = lltype.Struct("dictentry", *entryfields)
+    DICTENTRYARRAY = lltype.GcArray(DICTENTRY,
+                                    adtmeths=entrymeths)
+    fields =          [ ("num_items", lltype.Signed),
+                        ("num_used_items", lltype.Signed),
+                        ("resize_counter", lltype.Signed),
+                        ("indexes", llmemory.GCREF),
+                        ("lookup_function_no", lltype.Signed),
+                        ("entries", lltype.Ptr(DICTENTRYARRAY)) ]
+    if get_custom_eq_hash is not None:
+        r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash()
+        fields.extend([ ("fnkeyeq", r_rdict_eqfn.lowleveltype),
+                        ("fnkeyhash", r_rdict_hashfn.lowleveltype) ])
+        adtmeths = {
+            'keyhash':        ll_keyhash_custom,
+            'keyeq':          ll_keyeq_custom,
+            'r_rdict_eqfn':   r_rdict_eqfn,
+            'r_rdict_hashfn': r_rdict_hashfn,
+            'paranoia':       True,
+            }
+    else:
+        # figure out which functions must be used to hash and compare
+        ll_keyhash = ll_hash_function
+        ll_keyeq = ll_eq_function
+        ll_keyhash = lltype.staticAdtMethod(ll_keyhash)
+        if ll_keyeq is not None:
+            ll_keyeq = lltype.staticAdtMethod(ll_keyeq)
+        adtmeths = {
+            'keyhash':  ll_keyhash,
+            'keyeq':    ll_keyeq,
+            'paranoia': False,
+            }
+    adtmeths['KEY']   = DICTKEY
+    adtmeths['VALUE'] = DICTVALUE
+    adtmeths['lookup_function'] = lltype.staticAdtMethod(ll_call_lookup_function)
+    adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
+
+    family = LookupFamily()
+    adtmeths['lookup_family'] = family
+
+    DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
+                                *fields))
+
+    family.empty_array = DICTENTRYARRAY.allocate(0)
+    if setup_lookup_funcs:
+        _setup_lookup_funcs(DICT, rtyper, family)
+    return DICT
+
+def _setup_lookup_funcs(DICT, rtyper, family):
+    DICTKEY = DICT.entries.TO.OF.key
+    LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY,
+                                              lltype.Signed, lltype.Signed],
+                                              lltype.Signed))
+
+
+    STORECLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
+                                                  lltype.Signed,
+                                                  lltype.Signed],
+                                                 lltype.Void))
+
+    for name, T in [('byte', rffi.UCHAR),
+                    ('short', rffi.USHORT),
+                    ('int', rffi.UINT),
+                    ('long', lltype.Unsigned)]:
+        if name == 'int' and not IS_64BIT:
+            continue
+        lookupfn, storecleanfn = new_lookup_functions(LOOKUP_FUNC,
+                                                      STORECLEAN_FUNC, T=T,
+                                                      rtyper=rtyper)
+        setattr(family, '%s_lookup_function' % name, lookupfn)
+        setattr(family, '%s_insert_clean_function' % name, storecleanfn)
+
+def llhelper_or_compile(rtyper, FUNCPTR, ll_func):
+    # the check is for pseudo rtyper from tests
+    if rtyper is None or not hasattr(rtyper, 'annotate_helper_fn'):
+        return llhelper(FUNCPTR, ll_func)
+    else:
+        return rtyper.annotate_helper_fn(ll_func, FUNCPTR.TO.ARGS)
+
+class LookupFamily:
+    def _freeze_(self):
+        return True
+
+
+class OrderedDictRepr(AbstractDictRepr):
+
+    def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
+                 custom_eq_hash=None, force_non_null=False):
+        assert not force_non_null
+        self.rtyper = rtyper
+        self.finalized = False
+        self.DICT = lltype.GcForwardReference()
+        self.lowleveltype = lltype.Ptr(self.DICT)
+        self.custom_eq_hash = custom_eq_hash is not None
+        if not isinstance(key_repr, rmodel.Repr):  # not computed yet, done by setup()
+            assert callable(key_repr)
+            self._key_repr_computer = key_repr
+        else:
+            self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr)
+        if not isinstance(value_repr, rmodel.Repr):  # not computed yet, done by setup()
+            assert callable(value_repr)
+            self._value_repr_computer = value_repr
+        else:
+            self.external_value_repr, self.value_repr = self.pickrepr(value_repr)
+        self.dictkey = dictkey
+        self.dictvalue = dictvalue
+        self.dict_cache = {}
+        self._custom_eq_hash_repr = custom_eq_hash
+        # setup() needs to be called to finish this initialization
+
+    def _externalvsinternal(self, rtyper, item_repr):
+        return rmodel.externalvsinternal(self.rtyper, item_repr)
+
+    def _setup_repr(self):
+        if 'key_repr' not in self.__dict__:
+            key_repr = self._key_repr_computer()
+            self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr)
+        if 'value_repr' not in self.__dict__:
+            self.external_value_repr, self.value_repr = self.pickrepr(self._value_repr_computer())
+        if isinstance(self.DICT, lltype.GcForwardReference):
+            DICTKEY = self.key_repr.lowleveltype
+            DICTVALUE = self.value_repr.lowleveltype
+            # * we need an explicit flag if the key and the value is not
+            #   able to store dummy values
+            s_key   = self.dictkey.s_value
+            s_value = self.dictvalue.s_value
+            kwd = {}
+            if self.custom_eq_hash:
+                self.r_rdict_eqfn, self.r_rdict_hashfn = (
+                    self._custom_eq_hash_repr())
+                kwd['get_custom_eq_hash'] = self._custom_eq_hash_repr
+            else:
+                kwd['ll_hash_function'] = self.key_repr.get_ll_hash_function()
+                kwd['ll_eq_function'] = self.key_repr.get_ll_eq_function()
+                kwd['ll_fasthash_function'] = self.key_repr.get_ll_fasthash_function()
+            kwd['dummykeyobj'] = self.key_repr.get_ll_dummyval_obj(self.rtyper,
+                                                                   s_key)
+            kwd['dummyvalueobj'] = self.value_repr.get_ll_dummyval_obj(
+                self.rtyper, s_value)
+
+            kwd['setup_lookup_funcs'] = False
+            get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT,
+                        rtyper=self.rtyper, **kwd)
+
+    def _setup_repr_final(self):
+        if not self.finalized:
+            family = self.lowleveltype.TO.lookup_family
+            _setup_lookup_funcs(self.lowleveltype.TO, self.rtyper, family)
+            self.finalized = True
+
+
+    def convert_const(self, dictobj):
+        from rpython.rtyper.lltypesystem import llmemory
+        # get object from bound dict methods
+        #dictobj = getattr(dictobj, '__self__', dictobj)
+        if dictobj is None:
+            return lltype.nullptr(self.DICT)
+        if not isinstance(dictobj, (dict, objectmodel.r_dict)):
+            raise TypeError("expected a dict: %r" % (dictobj,))
+        try:
+            key = Constant(dictobj)
+            return self.dict_cache[key]
+        except KeyError:
+            self.setup()
+            self.setup_final()
+            l_dict = ll_newdict_size(self.DICT, len(dictobj))
+            self.dict_cache[key] = l_dict
+            r_key = self.key_repr
+            if r_key.lowleveltype == llmemory.Address:
+                raise TypeError("No prebuilt dicts of address keys")
+            r_value = self.value_repr
+            if isinstance(dictobj, objectmodel.r_dict):
+                if self.r_rdict_eqfn.lowleveltype != lltype.Void:
+                    l_fn = self.r_rdict_eqfn.convert_const(dictobj.key_eq)
+                    l_dict.fnkeyeq = l_fn
+                if self.r_rdict_hashfn.lowleveltype != lltype.Void:
+                    l_fn = self.r_rdict_hashfn.convert_const(dictobj.key_hash)
+                    l_dict.fnkeyhash = l_fn
+
+                for dictkeycontainer, dictvalue in dictobj._dict.items():
+                    llkey = r_key.convert_const(dictkeycontainer.key)
+                    llvalue = r_value.convert_const(dictvalue)
+                    _ll_dict_insertclean(l_dict, llkey, llvalue,
+                                         dictkeycontainer.hash)
+                return l_dict
+
+            else:
+                for dictkey, dictvalue in dictobj.items():
+                    llkey = r_key.convert_const(dictkey)
+                    llvalue = r_value.convert_const(dictvalue)
+                    _ll_dict_insertclean(l_dict, llkey, llvalue,
+                                         l_dict.keyhash(llkey))
+                return l_dict
+
+    def rtype_len(self, hop):
+        v_dict, = hop.inputargs(self)
+        return hop.gendirectcall(ll_dict_len, v_dict)
+
+    def rtype_bool(self, hop):
+        v_dict, = hop.inputargs(self)
+        return hop.gendirectcall(ll_dict_bool, v_dict)
+
+    def make_iterator_repr(self, *variant):
+        return DictIteratorRepr(self, *variant)
+
+    def rtype_method_get(self, hop):
+        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                 self.value_repr)
+        hop.exception_cannot_occur()
+        v_res = hop.gendirectcall(ll_dict_get, v_dict, v_key, v_default)
+        return self.recast_value(hop.llops, v_res)
+
+    def rtype_method_setdefault(self, hop):
+        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                 self.value_repr)
+        hop.exception_cannot_occur()
+        v_res = hop.gendirectcall(ll_dict_setdefault, v_dict, v_key, v_default)
+        return self.recast_value(hop.llops, v_res)
+
+    def rtype_method_copy(self, hop):
+        v_dict, = hop.inputargs(self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_copy, v_dict)
+
+    def rtype_method_update(self, hop):
+        v_dic1, v_dic2 = hop.inputargs(self, self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_update, v_dic1, v_dic2)
+
+    def _rtype_method_kvi(self, hop, ll_func):
+        v_dic, = hop.inputargs(self)
+        r_list = hop.r_result
+        cLIST = hop.inputconst(lltype.Void, r_list.lowleveltype.TO)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_func, cLIST, v_dic)
+
+    def rtype_method_keys(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_keys)
+
+    def rtype_method_values(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_values)
+
+    def rtype_method_items(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_items)
+
+    def rtype_method_iterkeys(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "keys").newiter(hop)
+
+    def rtype_method_itervalues(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "values").newiter(hop)
+
+    def rtype_method_iteritems(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "items").newiter(hop)
+
+    def rtype_method_clear(self, hop):
+        v_dict, = hop.inputargs(self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_clear, v_dict)
+
+    def rtype_method_popitem(self, hop):
+        v_dict, = hop.inputargs(self)
+        r_tuple = hop.r_result
+        cTUPLE = hop.inputconst(lltype.Void, r_tuple.lowleveltype)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_popitem, cTUPLE, v_dict)
+
+    def rtype_method_pop(self, hop):
+        if hop.nb_args == 2:
+            v_args = hop.inputargs(self, self.key_repr)
+            target = ll_dict_pop
+        elif hop.nb_args == 3:
+            v_args = hop.inputargs(self, self.key_repr, self.value_repr)
+            target = ll_dict_pop_default
+        hop.exception_is_here()
+        v_res = hop.gendirectcall(target, *v_args)
+        return self.recast_value(hop.llops, v_res)
+
+class __extend__(pairtype(OrderedDictRepr, rmodel.Repr)):
+
+    def rtype_getitem((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        if not r_dict.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)   # record that we know about it
+        hop.exception_is_here()
+        v_res = hop.gendirectcall(ll_dict_getitem, v_dict, v_key)
+        return r_dict.recast_value(hop.llops, v_res)
+
+    def rtype_delitem((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        if not r_dict.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)   # record that we know about it
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_delitem, v_dict, v_key)
+
+    def rtype_setitem((r_dict, r_key), hop):
+        v_dict, v_key, v_value = hop.inputargs(r_dict, r_dict.key_repr, r_dict.value_repr)
+        if r_dict.custom_eq_hash:
+            hop.exception_is_here()
+        else:
+            hop.exception_cannot_occur()
+        hop.gendirectcall(ll_dict_setitem, v_dict, v_key, v_value)
+
+    def rtype_contains((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_contains, v_dict, v_key)
+
+class __extend__(pairtype(OrderedDictRepr, OrderedDictRepr)):
+    def convert_from_to((r_dict1, r_dict2), v, llops):
+        # check that we don't convert from Dicts with
+        # different key/value types
+        if r_dict1.dictkey is None or r_dict2.dictkey is None:
+            return NotImplemented
+        if r_dict1.dictkey is not r_dict2.dictkey:
+            return NotImplemented
+        if r_dict1.dictvalue is None or r_dict2.dictvalue is None:
+            return NotImplemented
+        if r_dict1.dictvalue is not r_dict2.dictvalue:
+            return NotImplemented
+        return v
+
+# ____________________________________________________________
+#
+#  Low-level methods.  These can be run for testing, but are meant to
+#  be direct_call'ed from rtyped flow graphs, which means that they will
+#  get flowed and annotated, mostly with SomePtr.
+
+DICTINDEX_LONG = lltype.Ptr(lltype.GcArray(lltype.Unsigned))
+DICTINDEX_INT = lltype.Ptr(lltype.GcArray(rffi.UINT))
+DICTINDEX_SHORT = lltype.Ptr(lltype.GcArray(rffi.USHORT))
+DICTINDEX_BYTE = lltype.Ptr(lltype.GcArray(rffi.UCHAR))
+
+IS_64BIT = sys.maxint != 2 ** 31 - 1
+
+if IS_64BIT:
+    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG = range(4)
+else:
+    FUNC_BYTE, FUNC_SHORT, FUNC_LONG = range(3)
+
+def ll_malloc_indexes_and_choose_lookup(d, n):
+    if n <= 256:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_BYTE.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_BYTE
+    elif n <= 65536:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_SHORT.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_SHORT
+    elif IS_64BIT and n <= 2 ** 32:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_INT.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_INT
+    else:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_LONG.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_LONG
+
+def ll_call_insert_clean_function(d, hash, i):
+    DICT = lltype.typeOf(d).TO
+    if d.lookup_function_no == FUNC_BYTE:
+        DICT.lookup_family.byte_insert_clean_function(d, hash, i)
+    elif d.lookup_function_no == FUNC_SHORT:
+        DICT.lookup_family.short_insert_clean_function(d, hash, i)
+    elif IS_64BIT and d.lookup_function_no == FUNC_INT:
+        DICT.lookup_family.int_insert_clean_function(d, hash, i)
+    elif d.lookup_function_no == FUNC_LONG:
+        DICT.lookup_family.long_insert_clean_function(d, hash, i)
+    else:
+        assert False
+
+def ll_valid_from_flag(entries, i):
+    return entries[i].f_valid
+
+def ll_valid_from_key(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    return entries[i].key != dummy
+
+def ll_valid_from_value(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    return entries[i].value != dummy
+
+def ll_mark_deleted_in_flag(entries, i):
+    entries[i].f_valid = False
+
+def ll_mark_deleted_in_key(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    entries[i].key = dummy
+
+def ll_mark_deleted_in_value(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    entries[i].value = dummy
+
+def ll_hash_from_cache(entries, i):
+    return entries[i].f_hash
+
+def ll_hash_recomputed(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    return ENTRIES.fasthashfn(entries[i].key)
+
+def ll_keyhash_custom(d, key):
+    DICT = lltype.typeOf(d).TO
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+
+def ll_keyeq_custom(d, key1, key2):
+    DICT = lltype.typeOf(d).TO
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+
+def ll_dict_len(d):
+    return d.num_items
+
+def ll_dict_bool(d):
+    # check if a dict is True, allowing for None
+    return bool(d) and d.num_items != 0
+
+def ll_dict_getitem(d, key):
+    index = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
+    if index != -1:
+        return d.entries[index].value
+    else:
+        raise KeyError
+
+def ll_dict_setitem(d, key, value):
+    hash = d.keyhash(key)
+    index = d.lookup_function(d, key, hash, FLAG_STORE)
+    return _ll_dict_setitem_lookup_done(d, key, value, hash, index)
+
+# It may be safe to look inside always, it has a few branches though, and their
+# frequencies needs to be investigated.
+ at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
+def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    if i >= 0:
+        entry = d.entries[i]
+        entry.value = value
+    else:
+        if len(d.entries) == d.num_used_items:
+            if ll_dict_grow(d):
+                ll_call_insert_clean_function(d, hash, d.num_used_items)
+        entry = d.entries[d.num_used_items]
+        entry.key = key
+        entry.value = value
+        if hasattr(ENTRY, 'f_hash'):
+            entry.f_hash = hash
+        if hasattr(ENTRY, 'f_valid'):
+            entry.f_valid = True
+        d.num_used_items += 1
+        d.num_items += 1
+        rc = d.resize_counter - 3
+        if rc <= 0:
+            ll_dict_resize(d)
+            rc = d.resize_counter - 3
+            ll_assert(rc > 0, "ll_dict_resize failed?")
+        d.resize_counter = rc
+
+def _ll_dict_insertclean(d, key, value, hash):
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    ll_call_insert_clean_function(d, hash, d.num_used_items)
+    entry = d.entries[d.num_used_items]
+    entry.key = key
+    entry.value = value
+    if hasattr(ENTRY, 'f_hash'):
+        entry.f_hash = hash
+    if hasattr(ENTRY, 'f_valid'):
+        entry.f_valid = True
+    d.num_used_items += 1
+    d.num_items += 1
+    rc = d.resize_counter - 3
+    d.resize_counter = rc
+
+def _ll_len_of_d_indexes(d):
+    # xxx Haaaack: returns len(d.indexes).  Works independently of
+    # the exact type pointed to by d, using a forced cast...
+    return len(rffi.cast(DICTINDEX_BYTE, d.indexes))
+
+def _overallocate_entries_len(baselen):
+    # This over-allocates proportional to the list size, making room
+    # for additional growth.  The over-allocation is mild, but is
+    # enough to give linear-time amortized behavior over a long
+    # sequence of appends() in the presence of a poorly-performing
+    # system malloc().
+    # The growth pattern is:  0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
+    newsize = baselen + 1
+    if newsize < 9:
+        some = 3
+    else:
+        some = 6
+    some += newsize >> 3
+    return newsize + some
+
+ at jit.dont_look_inside
+def ll_dict_grow(d):
+    if d.num_items < d.num_used_items // 4:
+        ll_dict_remove_deleted_items(d)
+        return True
+
+    new_allocated = _overallocate_entries_len(len(d.entries))
+
+    # Detect an obscure case where the indexes numeric type is too
+    # small to store all the entry indexes
+    if (max(128, _ll_len_of_d_indexes(d)) - new_allocated
+                   < MIN_INDEXES_MINUS_ENTRIES):
+        ll_dict_remove_deleted_items(d)
+        return True
+
+    newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+    rgc.ll_arraycopy(d.entries, newitems, 0, 0, len(d.entries))
+    d.entries = newitems
+    return False
+
+def ll_dict_remove_deleted_items(d):
+    new_allocated = _overallocate_entries_len(d.num_items)
+    if new_allocated < len(d.entries) // 2:
+        newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+    else:
+        newitems = d.entries
+    #
+    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
+    isrc = 0
+    idst = 0
+    while isrc < len(d.entries):
+        if d.entries.valid(isrc):
+            src = d.entries[isrc]
+            dst = newitems[idst]
+            dst.key = src.key
+            dst.value = src.value
+            if hasattr(ENTRY, 'f_hash'):
+                dst.f_hash = src.f_hash
+            if hasattr(ENTRY, 'f_valid'):
+                assert src.f_valid
+                dst.f_valid = True
+            idst += 1
+        isrc += 1
+    d.entries = newitems
+    assert d.num_items == idst
+    d.num_used_items = idst
+
+    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+
+
+def ll_dict_delitem(d, key):
+    index = d.lookup_function(d, key, d.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        raise KeyError
+    _ll_dict_del(d, index)
+
+ at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
+def _ll_dict_del(d, index):
+    d.entries.mark_deleted(index)
+    d.num_items -= 1
+    # clear the key and the value if they are GC pointers
+    ENTRIES = lltype.typeOf(d.entries).TO
+    ENTRY = ENTRIES.OF
+    entry = d.entries[index]
+    if ENTRIES.must_clear_key:
+        entry.key = lltype.nullptr(ENTRY.key.TO)
+    if ENTRIES.must_clear_value:
+        entry.value = lltype.nullptr(ENTRY.value.TO)
+    #
+    # The rest is commented out: like CPython we no longer shrink the
+    # dictionary here.  It may shrink later if we try to append a number
+    # of new items to it.  Unsure if this behavior was designed in
+    # CPython or is accidental.  A design reason would be that if you
+    # delete all items in a dictionary (e.g. with a series of
+    # popitem()), then CPython avoids shrinking the table several times.
+    #num_entries = len(d.entries)
+    #if num_entries > DICT_INITSIZE and d.num_items <= num_entries / 4:
+    #    ll_dict_resize(d)
+    # A previous xxx: move the size checking and resize into a single
+    # call which is opaque to the JIT when the dict isn't virtual, to
+    # avoid extra branches.
+
+def ll_dict_resize(d):
+    # make a 'new_size' estimate and shrink it if there are many
+    # deleted entry markers.  See CPython for why it is a good idea to
+    # quadruple the dictionary size as long as it's not too big.
+    num_items = d.num_items
+    if num_items > 50000:
+        new_estimate = num_items * 2
+    else:
+        new_estimate = num_items * 4
+    new_size = DICT_INITSIZE
+    while new_size <= new_estimate:
+        new_size *= 2
+
+    if new_size < _ll_len_of_d_indexes(d):
+        ll_dict_remove_deleted_items(d)
+    else:
+        ll_dict_reindex(d, new_size)
+ll_dict_resize.oopspec = 'dict.resize(d)'
+
+def ll_dict_reindex(d, new_size):
+    ll_malloc_indexes_and_choose_lookup(d, new_size)
+    d.resize_counter = new_size * 2 - d.num_items * 3
+    assert d.resize_counter > 0
+    #
+    entries = d.entries
+    i = 0
+    while i < d.num_used_items:
+        if entries.valid(i):
+            hash = entries.hash(i)
+            ll_call_insert_clean_function(d, hash, i)
+        i += 1
+    #old_entries.delete() XXXX!
+
+# ------- a port of CPython's dictobject.c's lookdict implementation -------
+PERTURB_SHIFT = 5
+
+FREE = 0
+DELETED = 1
+VALID_OFFSET = 2
+MIN_INDEXES_MINUS_ENTRIES = VALID_OFFSET + 1
+
+FLAG_LOOKUP = 0
+FLAG_STORE = 1
+FLAG_DELETE = 2
+FLAG_DELETE_TRY_HARD = 3
+
+def new_lookup_functions(LOOKUP_FUNC, STORECLEAN_FUNC, T, rtyper=None):
+    INDEXES = lltype.Ptr(lltype.GcArray(T))
+
+    def ll_kill_something(d):
+        i = 0
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        while True:
+            index = rffi.cast(lltype.Signed, indexes[i])
+            if index >= VALID_OFFSET:
+                indexes[i] = rffi.cast(T, DELETED)
+                return index
+            i += 1
+
+    @jit.look_inside_iff(lambda d, key, hash, store_flag:
+                         jit.isvirtual(d) and jit.isconstant(key))
+    def ll_dict_lookup(d, key, hash, store_flag):
+        entries = d.entries
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        mask = len(indexes) - 1
+        i = r_uint(hash & mask)
+        # do the first try before any looping
+        ENTRIES = lltype.typeOf(entries).TO
+        direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
+        index = rffi.cast(lltype.Signed, indexes[intmask(i)])
+        if index >= VALID_OFFSET:
+            checkingkey = entries[index - VALID_OFFSET].key
+            if direct_compare and checkingkey == key:
+                if store_flag == FLAG_DELETE:
+                    indexes[i] = rffi.cast(T, DELETED)
+                return index - VALID_OFFSET   # found the entry
+            if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
+                # correct hash, maybe the key is e.g. a different pointer to
+                # an equal object
+                found = d.keyeq(checkingkey, key)
+                #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found)
+                if d.paranoia:
+                    if (entries != d.entries or lltype.cast_opaque_ptr(llmemory.GCREF, indexes) != d.indexes or
+                        not entries.valid(index - VALID_OFFSET) or
+                        entries[index - VALID_OFFSET].key != checkingkey):
+                        # the compare did major nasty stuff to the dict: start over
+                        return ll_dict_lookup(d, key, hash, store_flag)
+                if found:
+                    if store_flag == FLAG_DELETE:
+                        indexes[i] = rffi.cast(T, DELETED)
+                    return index - VALID_OFFSET
+            deletedslot = -1
+        elif index == DELETED:
+            deletedslot = intmask(i)
+        else:
+            # pristine entry -- lookup failed
+            if store_flag == FLAG_STORE:
+                indexes[i] = rffi.cast(T, d.num_used_items + VALID_OFFSET)
+            elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
+                return ll_kill_something(d)
+            return -1
+
+        # In the loop, a deleted entry (everused and not valid) is by far
+        # (factor of 100s) the least likely outcome, so test for that last.
+        perturb = r_uint(hash)
+        while 1:
+            # compute the next index using unsigned arithmetic
+            i = (i << 2) + i + perturb + 1
+            i = i & mask
+            index = rffi.cast(lltype.Signed, indexes[intmask(i)])
+            if index == FREE:
+                if store_flag == FLAG_STORE:
+                    if deletedslot == -1:
+                        deletedslot = intmask(i)
+                    indexes[deletedslot] = rffi.cast(T, d.num_used_items +
+                                                     VALID_OFFSET)
+                elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
+                    return ll_kill_something(d)
+                return -1
+            elif index >= VALID_OFFSET:
+                checkingkey = entries[index - VALID_OFFSET].key
+                if direct_compare and checkingkey == key:
+                    if store_flag == FLAG_DELETE:
+                        indexes[i] = rffi.cast(T, DELETED)
+                    return index - VALID_OFFSET   # found the entry
+                if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
+                    # correct hash, maybe the key is e.g. a different pointer to
+                    # an equal object
+                    found = d.keyeq(checkingkey, key)
+                    if d.paranoia:
+                        if (entries != d.entries or lltype.cast_opaque_ptr(llmemory.GCREF, indexes) != d.indexes or
+                            not entries.valid(index - VALID_OFFSET) or
+                            entries[index - VALID_OFFSET].key != checkingkey):
+                            # the compare did major nasty stuff to the dict: start over
+                            return ll_dict_lookup(d, key, hash, store_flag)
+                    if found:
+                        if store_flag == FLAG_DELETE:
+                            indexes[i] = rffi.cast(T, DELETED)
+                        return index - VALID_OFFSET
+            elif deletedslot == -1:
+                deletedslot = intmask(i)
+            perturb >>= PERTURB_SHIFT
+
+    def ll_dict_store_clean(d, hash, index):
+        # a simplified version of ll_dict_lookup() which assumes that the
+        # key is new, and the dictionary doesn't contain deleted entries.
+        # It only finds the next free slot for the given hash.
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        mask = len(indexes) - 1
+        i = r_uint(hash & mask)
+        perturb = r_uint(hash)
+        while rffi.cast(lltype.Signed, indexes[i]) != 0:
+            i = (i << 2) + i + perturb + 1
+            i = i & mask
+            perturb >>= PERTURB_SHIFT
+        indexes[i] = rffi.cast(T, index + VALID_OFFSET)
+
+    return (llhelper_or_compile(rtyper, LOOKUP_FUNC, ll_dict_lookup),
+            llhelper_or_compile(rtyper, STORECLEAN_FUNC, ll_dict_store_clean))
+
+# ____________________________________________________________
+#
+#  Irregular operations.
+
+DICT_INITSIZE = 8
+
+def ll_newdict(DICT):
+    d = DICT.allocate()
+    d.entries = DICT.lookup_family.empty_array
+    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = DICT_INITSIZE * 2
+    return d
+
+def ll_newdict_size(DICT, orig_length_estimate):
+    length_estimate = (orig_length_estimate // 2) * 3
+    n = DICT_INITSIZE
+    while n < length_estimate:
+        n *= 2
+    d = DICT.allocate()
+    d.entries = DICT.entries.TO.allocate(orig_length_estimate)
+    ll_malloc_indexes_and_choose_lookup(d, n)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = n * 2
+    return d
+
+# rpython.memory.lldict uses a dict based on Struct and Array
+# instead of GcStruct and GcArray, which is done by using different
+# 'allocate' and 'delete' adtmethod implementations than the ones below
+def _ll_malloc_dict(DICT):
+    return lltype.malloc(DICT)
+def _ll_malloc_entries(ENTRIES, n):
+    return lltype.malloc(ENTRIES, n, zero=True)
+def _ll_free_entries(entries):
+    pass
+
+
+def rtype_r_dict(hop):
+    r_dict = hop.r_result
+    if not r_dict.custom_eq_hash:
+        raise TyperError("r_dict() call does not return an r_dict instance")
+    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
+    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
+    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
+    hop.exception_cannot_occur()
+    v_result = hop.gendirectcall(ll_newdict, cDICT)
+    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
+        hop.genop('setfield', [v_result, cname, v_eqfn])
+    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
+        hop.genop('setfield', [v_result, cname, v_hashfn])
+    return v_result
+
+# ____________________________________________________________
+#
+#  Iteration.
+
+def get_ll_dictiter(DICTPTR):
+    return lltype.Ptr(lltype.GcStruct('dictiter',
+                                      ('dict', DICTPTR),
+                                      ('index', lltype.Signed)))
+
+class DictIteratorRepr(AbstractDictIteratorRepr):
+
+    def __init__(self, r_dict, variant="keys"):
+        self.r_dict = r_dict
+        self.variant = variant
+        self.lowleveltype = get_ll_dictiter(r_dict.lowleveltype)
+        self.ll_dictiter = ll_dictiter
+        self.ll_dictnext = ll_dictnext_group[variant]
+
+
+def ll_dictiter(ITERPTR, d):
+    iter = lltype.malloc(ITERPTR.TO)
+    iter.dict = d
+    iter.index = 0
+    return iter
+
+def _make_ll_dictnext(kind):
+    # make three versions of the following function: keys, values, items
+    @jit.look_inside_iff(lambda RETURNTYPE, iter: jit.isvirtual(iter)
+                         and (iter.dict is None or
+                              jit.isvirtual(iter.dict)))
+    @jit.oopspec("dictiter.next%s(iter)" % kind)
+    def ll_dictnext(RETURNTYPE, iter):
+        # note that RETURNTYPE is None for keys and values
+        dict = iter.dict
+        if not dict:
+            raise StopIteration
+
+        entries = dict.entries
+        index = iter.index
+        assert index >= 0
+        entries_len = dict.num_used_items
+        while index < entries_len:
+            entry = entries[index]
+            is_valid = entries.valid(index)
+            index = index + 1
+            if is_valid:
+                iter.index = index
+                if RETURNTYPE is lltype.Void:
+                    return None
+                elif kind == 'items':
+                    r = lltype.malloc(RETURNTYPE.TO)
+                    r.item0 = recast(RETURNTYPE.TO.item0, entry.key)
+                    r.item1 = recast(RETURNTYPE.TO.item1, entry.value)
+                    return r
+                elif kind == 'keys':
+                    return entry.key
+                elif kind == 'values':
+                    return entry.value
+
+        # clear the reference to the dict and prevent restarts
+        iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
+        raise StopIteration
+
+    return ll_dictnext
+
+ll_dictnext_group = {'keys'  : _make_ll_dictnext('keys'),
+                     'values': _make_ll_dictnext('values'),
+                     'items' : _make_ll_dictnext('items')}
+
+# _____________________________________________________________
+# methods
+
+def ll_dict_get(dict, key, default):
+    index = dict.lookup_function(dict, key, dict.keyhash(key), FLAG_LOOKUP)
+    if index == -1:
+        return default
+    else:
+        return dict.entries[index].value
+
+def ll_dict_setdefault(dict, key, default):
+    hash = dict.keyhash(key)
+    index = dict.lookup_function(dict, key, hash, FLAG_STORE)
+    if index == -1:
+        _ll_dict_setitem_lookup_done(dict, key, default, hash, -1)
+        return default
+    else:
+        return dict.entries[index].value
+
+def ll_dict_copy(dict):
+    DICT = lltype.typeOf(dict).TO
+    newdict = DICT.allocate()
+    newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
+
+    newdict.num_items = dict.num_items
+    newdict.num_used_items = dict.num_used_items
+    if hasattr(DICT, 'fnkeyeq'):
+        newdict.fnkeyeq = dict.fnkeyeq
+    if hasattr(DICT, 'fnkeyhash'):
+        newdict.fnkeyhash = dict.fnkeyhash
+
+    i = 0
+    while i < newdict.num_used_items:
+        d_entry = newdict.entries[i]
+        entry = dict.entries[i]
+        ENTRY = lltype.typeOf(newdict.entries).TO.OF
+        d_entry.key = entry.key
+        if hasattr(ENTRY, 'f_valid'):
+            d_entry.f_valid = entry.f_valid
+        d_entry.value = entry.value
+        if hasattr(ENTRY, 'f_hash'):
+            d_entry.f_hash = entry.f_hash
+        i += 1
+
+    ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
+    return newdict
+ll_dict_copy.oopspec = 'dict.copy(dict)'
+
+def ll_dict_clear(d):
+    if d.num_used_items == 0:
+        return
+    DICT = lltype.typeOf(d).TO
+    old_entries = d.entries
+    d.entries = DICT.lookup_family.empty_array
+    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = DICT_INITSIZE * 2
+    # old_entries.delete() XXX
+ll_dict_clear.oopspec = 'dict.clear(d)'
+
+def ll_dict_update(dic1, dic2):
+    i = 0
+    while i < dic2.num_used_items:
+        entries = dic2.entries
+        if entries.valid(i):
+            entry = entries[i]
+            hash = entries.hash(i)
+            key = entry.key
+            value = entry.value
+            index = dic1.lookup_function(dic1, key, hash, FLAG_STORE)
+            _ll_dict_setitem_lookup_done(dic1, key, value, hash, index)
+        i += 1
+ll_dict_update.oopspec = 'dict.update(dic1, dic2)'
+
+# this is an implementation of keys(), values() and items()
+# in a single function.
+# note that by specialization on func, three different
+# and very efficient functions are created.
+
+def recast(P, v):
+    if isinstance(P, lltype.Ptr):
+        return lltype.cast_pointer(P, v)
+    else:
+        return v
+
+def _make_ll_keys_values_items(kind):
+    def ll_kvi(LIST, dic):
+        res = LIST.ll_newlist(dic.num_items)
+        entries = dic.entries
+        dlen = dic.num_used_items
+        items = res.ll_items()
+        i = 0
+        p = 0
+        while i < dlen:
+            if entries.valid(i):
+                ELEM = lltype.typeOf(items).TO.OF
+                if ELEM is not lltype.Void:
+                    entry = entries[i]
+                    if kind == 'items':
+                        r = lltype.malloc(ELEM.TO)
+                        r.item0 = recast(ELEM.TO.item0, entry.key)
+                        r.item1 = recast(ELEM.TO.item1, entry.value)
+                        items[p] = r
+                    elif kind == 'keys':
+                        items[p] = recast(ELEM, entry.key)
+                    elif kind == 'values':
+                        items[p] = recast(ELEM, entry.value)
+                p += 1
+            i += 1
+        assert p == res.ll_length()
+        return res
+    ll_kvi.oopspec = 'dict.%s(dic)' % kind
+    return ll_kvi
+
+ll_dict_keys   = _make_ll_keys_values_items('keys')
+ll_dict_values = _make_ll_keys_values_items('values')
+ll_dict_items  = _make_ll_keys_values_items('items')
+
+def ll_dict_contains(d, key):
+    i = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
+    return i != -1
+
+def _ll_getnextitem(dic):
+    if dic.num_items == 0:
+        raise KeyError
+
+    entries = dic.entries
+
+    while True:
+        i = dic.num_used_items - 1
+        if entries.valid(i):
+            break
+        dic.num_used_items -= 1
+
+    key = entries[i].key
+    index = dic.lookup_function(dic, key, entries.hash(i),
+                                FLAG_DELETE_TRY_HARD)
+    # if the lookup function returned me a random strange thing,
+    # don't care about deleting the item
+    if index == dic.num_used_items - 1:
+        dic.num_used_items -= 1
+    else:
+        assert index != -1
+    return index
+
+def ll_dict_popitem(ELEM, dic):
+    i = _ll_getnextitem(dic)
+    entry = dic.entries[i]
+    r = lltype.malloc(ELEM.TO)
+    r.item0 = recast(ELEM.TO.item0, entry.key)
+    r.item1 = recast(ELEM.TO.item1, entry.value)
+    _ll_dict_del(dic, i)
+    return r
+
+def ll_dict_pop(dic, key):
+    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        raise KeyError
+    value = dic.entries[index].value
+    _ll_dict_del(dic, index)
+    return value
+
+def ll_dict_pop_default(dic, key, dfl):
+    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        return dfl
+    value = dic.entries[index].value
+    _ll_dict_del(dic, index)
+    return value
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -1,3 +1,5 @@
+from collections import OrderedDict
+
 from rpython.annotator import model as annmodel
 from rpython.flowspace.model import Constant
 from rpython.rlib import rarithmetic, objectmodel
@@ -726,10 +728,29 @@
 
     raise TyperError("hasattr is only suported on a constant")
 
+def rtype_ordered_dict(hop):
+    from rpython.rtyper.lltypesystem.rordereddict import ll_newdict
+
+    hop.exception_cannot_occur()
+    r_dict = hop.r_result
+    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
+    v_result = hop.gendirectcall(ll_newdict, cDICT)
+    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
+    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
+    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
+        hop.genop('setfield', [v_result, cname, v_eqfn])
+    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
+        hop.genop('setfield', [v_result, cname, v_hashfn])
+    return v_result
+
 BUILTIN_TYPER[objectmodel.instantiate] = rtype_instantiate
 BUILTIN_TYPER[isinstance] = rtype_builtin_isinstance
 BUILTIN_TYPER[hasattr] = rtype_builtin_hasattr
 BUILTIN_TYPER[objectmodel.r_dict] = rtype_r_dict
+BUILTIN_TYPER[OrderedDict] = rtype_ordered_dict
+BUILTIN_TYPER[objectmodel.r_ordereddict] = rtype_ordered_dict
 
 # _________________________________________________________________
 # weakrefs
diff --git a/rpython/rtyper/rdict.py b/rpython/rtyper/rdict.py
--- a/rpython/rtyper/rdict.py
+++ b/rpython/rtyper/rdict.py
@@ -4,8 +4,12 @@
 
 
 class __extend__(annmodel.SomeDict):
+    def get_dict_repr(self):
+        from rpython.rtyper.lltypesystem.rdict import DictRepr
+
+        return DictRepr
+
     def rtyper_makerepr(self, rtyper):
-        from rpython.rtyper.lltypesystem.rdict import DictRepr
         dictkey = self.dictdef.dictkey
         dictvalue = self.dictdef.dictvalue
         s_key = dictkey.s_value
@@ -16,7 +20,7 @@
                                       rtyper.getrepr(dictkey.s_rdict_hashfn))
         else:
             custom_eq_hash = None
-        return DictRepr(rtyper, lambda: rtyper.getrepr(s_key),
+        return self.get_dict_repr()(rtyper, lambda: rtyper.getrepr(s_key),
                         lambda: rtyper.getrepr(s_value), dictkey, dictvalue,
                         custom_eq_hash, force_non_null)
 
@@ -25,6 +29,11 @@
         self.dictdef.dictvalue.dont_change_any_more = True
         return (self.__class__, self.dictdef.dictkey, self.dictdef.dictvalue)
 
+class __extend__(annmodel.SomeOrderedDict):
+    def get_dict_repr(self):
+        from rpython.rtyper.lltypesystem.rordereddict import OrderedDictRepr
+
+        return OrderedDictRepr
 
 class AbstractDictRepr(rmodel.Repr):
 
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -22,11 +22,11 @@
         yield x
 
 
-class TestRdict(BaseRtypingTest):
-
+class BaseTestRDict(BaseRtypingTest):
     def test_dict_creation(self):
         def createdict(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             return d['hello']
 
         res = self.interpret(createdict, [42])
@@ -34,7 +34,8 @@
 
     def test_dict_getitem_setitem(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             d['world'] = i + 1
             return d['hello'] * d['world']
         res = self.interpret(func, [6])
@@ -42,7 +43,8 @@
 
     def test_dict_getitem_keyerror(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             try:
                 return d['world']
             except KeyError:
@@ -52,7 +54,8 @@
 
     def test_dict_del_simple(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             d['world'] = i + 1
             del d['hello']
             return len(d)
@@ -61,7 +64,8 @@
 
     def test_dict_clear(self):
         def func(i):
-            d = {'abc': i}
+            d = self.newdict()
+            d['abc'] = i
             d['def'] = i+1
             d.clear()
             d['ghi'] = i+2
@@ -72,7 +76,8 @@
 
     def test_empty_strings(self):
         def func(i):
-            d = {'' : i}
+            d = self.newdict()
+            d[''] = i
             del d['']
             try:
                 d['']
@@ -84,7 +89,8 @@
         assert res == 1
 
         def func(i):
-            d = {'' : i}
+            d = self.newdict()
+            d[''] = i
             del d['']
             d[''] = i + 1
             return len(d)
@@ -94,9 +100,10 @@
     def test_dict_bool(self):
         def func(i):
             if i:
-                d = {}
+                d = self.newdict()
             else:
-                d = {i: i+1}
+                d = self.newdict()
+                d[i] = i+1
             if d:
                 return i
             else:
@@ -106,17 +113,20 @@
 
     def test_contains(self):
         def func(x, y):
-            d = {x: x+1}
+            d = self.newdict()
+            d[x] = x+1
             return y in d
         assert self.interpret(func, [42, 0]) == False
         assert self.interpret(func, [42, 42]) == True
 
     def test_contains_2(self):
-        d = {'5': None, '7': None}
+        d = self.newdict()
+        d['5'] = None
+        d['7'] = None
         def func(x):
             return chr(x) in d
-        #assert self.interpret(func, [ord('5')]) == True
-        #assert self.interpret(func, [ord('6')]) == False
+        assert self.interpret(func, [ord('5')]) == True
+        assert self.interpret(func, [ord('6')]) == False
 
         def func(n):
             return str(n) in d
@@ -124,7 +134,7 @@
 
     def test_dict_iteration(self):
         def func(i, j):
-            d = {}
+            d = self.newdict()
             d['hello'] = i
             d['world'] = j
             k = 1
@@ -136,7 +146,7 @@
 
     def test_dict_itermethods(self):
         def func():
-            d = {}
+            d = self.newdict()
             d['hello'] = 6
             d['world'] = 7
             k1 = k2 = k3 = 1
@@ -151,19 +161,9 @@
         res = self.interpret(func, [])
         assert res == 42 + 42 + 42
 
-    def test_two_dicts_with_different_value_types(self):
-        def func(i):
-            d1 = {}
-            d1['hello'] = i + 1
-            d2 = {}
-            d2['world'] = d1
-            return d2['world']['hello']
-        res = self.interpret(func, [5])
-        assert res == 6
-
     def test_dict_get(self):
         def func():
-            dic = {}
+            dic = self.newdict()
             x1 = dic.get('hi', 42)
             dic['blah'] = 1 # XXX this triggers type determination
             x2 = dic.get('blah', 2)
@@ -174,7 +174,7 @@
     def test_dict_get_empty(self):
         def func():
             # this time without writing to the dict
-            dic = {}
+            dic = self.newdict()
             x1 = dic.get('hi', 42)
             x2 = dic.get('blah', 2)
             return x1 * 10 + x2
@@ -183,14 +183,14 @@
 
     def test_dict_setdefault(self):
         def f():
-            d = {}
+            d = self.newdict()
             d.setdefault('a', 2)
             return d['a']
         res = self.interpret(f, ())
         assert res == 2
 
         def f():
-            d = {}
+            d = self.newdict()
             d.setdefault('a', 2)
             x = d.setdefault('a', -3)
             return x
@@ -200,7 +200,9 @@
     def test_dict_copy(self):
         def func():
             # XXX this does not work if we use chars, only!
-            dic = {'ab':1, 'b':2}
+            dic = self.newdict()
+            dic['ab'] = 1
+            dic['b'] = 2
             d2 = dic.copy()
             ok = 1
             for key in d2:
@@ -215,8 +217,12 @@
 
     def test_dict_update(self):
         def func():
-            dic = {'ab':1000, 'b':200}
-            d2 = {'b':30, 'cb':4}
+            dic = self.newdict()
+            dic['ab'] = 1000
+            dic['b'] = 200
+            d2 = self.newdict()
+            d2['b'] = 30
+            d2['cb'] = 4
             dic.update(d2)
             ok = len(dic) == 3
             sum = ok
@@ -228,7 +234,9 @@
 
     def test_dict_keys(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             keys = dic.keys()
             return ord(keys[0][1]) + ord(keys[1][1]) - 2*ord('0') + len(keys)
         res = self.interpret(func, ())#, view=True)
@@ -240,8 +248,11 @@
         class A(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {A(): 1, A(): 2}
+            dic0 = self.newdict()
+            dic0[Empty()] = 2
+            dic = self.newdict()
+            dic[A()] = 1
+            dic[A()] = 2
             keys = dic.keys()
             return (isinstance(keys[1], A))*2+(isinstance(keys[0],A))
         res = self.interpret(func, [])
@@ -253,8 +264,11 @@
         class A(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {A(): 1, A(): 2}
+            dic0 = self.newdict()
+            dic0[Empty()] = 2
+            dic = self.newdict()
+            dic[A()] = 1
+            dic[A()] = 2
             a = 0
             for k in dic.iterkeys():
                 a += isinstance(k, A)
@@ -264,7 +278,9 @@
 
     def test_dict_values(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             values = dic.values()
             return values[0] + values[1] + len(values)
         res = self.interpret(func, ())
@@ -274,7 +290,9 @@
         class A:
             pass
         def func():
-            dic = {1: A(), 2: A()}
+            dic = self.newdict()
+            dic[1] = A()
+            dic[2] = A()
             vals = dic.values()
             return (isinstance(vals[1], A))*2+(isinstance(vals[0],A))
         res = self.interpret(func, [])
@@ -284,7 +302,9 @@
         class A:
             pass
         def func():
-            dic = {1: A(), 2: A()}
+            dic = self.newdict()
+            dic[1] = A()
+            dic[2] = A()
             a = 0
             for v in dic.itervalues():
                 a += isinstance(v, A)
@@ -300,8 +320,11 @@
         class B(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {B(): A(), B(): A()}
+            dic0 = self.newdict()
+            dic0[Empty()] = A()
+            dic = self.newdict()
+            dic[B()] = A()
+            dic[B()] = A()
             items = dic.items()
             b = 0
             a = 0
@@ -320,8 +343,11 @@
         class B(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {B(): A(), B(): A()}
+            dic0 = self.newdict()
+            dic0[Empty()] = A()
+            dic = self.newdict()
+            dic[B()] = A()
+            dic[B()] = A()
             b = 0
             a = 0
             for k, v in dic.iteritems():
@@ -333,7 +359,9 @@
 
     def test_dict_items(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             items = dic.items()
             res = len(items)
             for key, value in items:
@@ -344,13 +372,17 @@
 
     def test_dict_contains(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             return ' 4' in dic and ' 9' not in dic
         res = self.interpret(func, ())
         assert res is True
 
     def test_dict_contains_with_constant_dict(self):
-        dic = {'4':1000, ' 8':200}
+        dic = self.newdict()
+        dic['4'] = 1000
+        dic['8'] = 200
         def func(i):
             return chr(i) in dic
         res = self.interpret(func, [ord('4')])
@@ -367,7 +399,9 @@
             a = A()
             a.d = None
             if n > 0:
-                a.d = {str(n): 1, "42": 2}
+                a.d = self.newdict()
+                a.d[str(n)] = 1
+                a.d["42"] = 2
                 del a.d["42"]
             return negate(a.d)
         res = self.interpret(func, [10])
@@ -379,7 +413,8 @@
 
     def test_int_dict(self):
         def func(a, b):
-            dic = {12: 34}
+            dic = self.newdict()
+            dic[12] = 34
             dic[a] = 1000
             return dic.get(b, -123)
         res = self.interpret(func, [12, 12])
@@ -403,7 +438,7 @@
         def f():
             a = A()
             b = B()
-            d = {}
+            d = self.newdict()
             d[b] = 7
             d[a] = 3
             return len(d) + d[a] + d[b]
@@ -411,7 +446,9 @@
         assert res == 12
 
     def test_captured_get(self):
-        get = {1:2}.get
+        d = self.newdict()
+        d[1] = 2
+        get = d.get
         def f():
             return get(1, 3)+get(2, 4)
         res = self.interpret(f, [])
@@ -431,40 +468,21 @@
         def f():
             lst = [A()]
             res1 = A() in lst
-            d2 = {B(): None, B(): None}
+            d2 = self.newdict()
+            d2[B()] = None
+            d2[B()] = None
             return res1+len(d2)
         res = self.interpret(f, [])
         assert res == 2
 
-
-    def test_type_erase(self):
-        class A(object):
-            pass
-        class B(object):
-            pass
-
-        def f():
-            return {A(): B()}, {B(): A()}
-
-        t = TranslationContext()
-        s = t.buildannotator().build_types(f, [])
-        rtyper = t.buildrtyper()
-        rtyper.specialize()
-
-        s_AB_dic = s.items[0]
-        s_BA_dic = s.items[1]
-
-        r_AB_dic = rtyper.getrepr(s_AB_dic)
-        r_BA_dic = rtyper.getrepr(s_AB_dic)
-
-        assert r_AB_dic.lowleveltype == r_BA_dic.lowleveltype
-
     def test_identity_hash_is_fast(self):
         class A(object):
             pass
 
         def f():
-            return {A(): 1}
+            d = self.newdict()
+            d[A()] = 1
+            return d
 
         t = TranslationContext()
         s = t.buildannotator().build_types(f, [])
@@ -476,7 +494,7 @@
 
     def test_tuple_dict(self):
         def f(i):
-            d = {}
+            d = self.newdict()
             d[(1, 4.5, (str(i), 2), 2)] = 4
             d[(1, 4.5, (str(i), 2), 3)] = 6
             return d[(1, 4.5, (str(i), 2), i)]
@@ -486,9 +504,9 @@
 
     def test_dict_of_dict(self):
         def f(n):
-            d = {}
+            d = self.newdict()
             d[5] = d
-            d[6] = {}
+            d[6] = self.newdict()
             return len(d[n])
 
         res = self.interpret(f, [5])
@@ -504,10 +522,9 @@
             pass
 
         def f(i):
-            d = {
-                A: 3,
-                B: 4,
-            }
+            d = self.newdict()
+            d[A] = 3
+            d[B] = 4
             if i:
                 cls = A
             else:
@@ -526,7 +543,9 @@
         class B(A):
             pass
 
-        d = {(A, 3): 3, (B, 0): 4}
+        d = self.newdict()
+        d[(A, 3)] = 3
+        d[(B, 0)] = 4
 
         def f(i):
             if i:
@@ -553,7 +572,9 @@
                 return 42
             return -1
         def g(n):
-            d = {1: n, 2: 2*n}
+            d = self.newdict()
+            d[1] = n
+            d[2] = 2*n
             return f(d)
         res = self.interpret(g, [3])
         assert res == 6
@@ -566,51 +587,19 @@
                 return 42
             return -1
         def g(n):
-            d = {1: n}
+            d = self.newdict()
+            d[1] = n
             f(d)
             return d[2]
         res = self.interpret(g, [3])
         assert res == 77
 
-    def test_r_dict(self):
-        class FooError(Exception):
-            pass
-        def myeq(n, m):
-            return n == m
-        def myhash(n):
-            if n < 0:
-                raise FooError
-            return -n
-        def f(n):
-            d = r_dict(myeq, myhash)
-            for i in range(10):
-                d[i] = i*i
-            try:
-                value1 = d[n]
-            except FooError:
-                value1 = 99
-            try:
-                value2 = n in d
-            except FooError:
-                value2 = 99
-            try:
-                value3 = d[-n]
-            except FooError:
-                value3 = 99
-            try:
-                value4 = (-n) in d
-            except FooError:
-                value4 = 99
-            return (value1 * 1000000 +
-                    value2 * 10000 +
-                    value3 * 100 +
-                    value4)
-        res = self.interpret(f, [5])
-        assert res == 25019999
-
     def test_resize_during_iteration(self):
         def func():
-            d = {5: 1, 6: 2, 7: 3}
+            d = self.newdict()
+            d[5] = 1
+            d[6] = 2
+            d[7] = 3
             try:
                 for key, value in d.iteritems():
                     d[key^16] = value*2
@@ -625,14 +614,21 @@
 
     def test_change_during_iteration(self):
         def func():
-            d = {'a': 1, 'b': 2}
+            d = self.newdict()
+            d['a'] = 1
+            d['b'] = 2
             for key in d:
                 d[key] = 42
             return d['a']
         assert self.interpret(func, []) == 42
 
     def test_dict_of_floats(self):
-        d = {3.0: 42, 3.1: 43, 3.2: 44, 3.3: 45, 3.4: 46}
+        d = self.newdict()
+        d[3.0] = 42
+        d[3.1] = 43
+        d[3.2] = 44
+        d[3.3] = 45
+        d[3.4] = 46
         def fn(f):
             return d[f]
 
@@ -643,7 +639,9 @@
         for r_t in [r_uint, r_longlong, r_ulonglong]:
             if r_t is r_int:
                 continue    # for 64-bit platforms: skip r_longlong
-            d = {r_t(2): 3, r_t(4): 5}
+            d = self.newdict()
+            d[r_t(2)] = 3
+            d[r_t(4)] = 5

From noreply at buildbot.pypy.org  Tue Oct 29 21:50:47 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 21:50:47 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: port test_{arrayops, complex,
	sorting, ufuncs}
Message-ID: <20131029205047.2F01B1C0163@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67706:fb907eaa46f8
Date: 2013-10-29 16:38 -0400
http://bitbucket.org/pypy/pypy/changeset/fb907eaa46f8/

Log:	port test_{arrayops,complex,sorting,ufuncs}

diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -1,2 +1,4 @@
 from _numpypy.multiarray import *
 from _numpypy.umath import *
+
+ufunc = type(sin)
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
--- a/pypy/module/micronumpy/test/test_arrayops.py
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -85,12 +85,10 @@
         assert c == 12.0
 
     def test_choose_basic(self):
-        from numpypy import array, choose
+        from numpypy import array
         a, b, c = array([1, 2, 3]), array([4, 5, 6]), array([7, 8, 9])
         r = array([2, 1, 0]).choose([a, b, c])
         assert (r == [7, 5, 3]).all()
-        r = choose(array([2, 1, 0]), [a, b, c])
-        assert (r == [7, 5, 3]).all()
 
     def test_choose_broadcast(self):
         from numpypy import array
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -130,9 +130,9 @@
             cls.w_c_pow = cls.space.wrap(interp2app(cls_c_pow))
 
     def test_fabs(self):
-        from numpypy import fabs, complex128
+        from numpypy import fabs, dtype
 
-        a = complex128(complex(-5., 5.))
+        a = dtype('complex128').type(complex(-5., 5.))
         raises(TypeError, fabs, a)
 
     def test_fmax(self):
@@ -194,7 +194,7 @@
         raises(TypeError, signbit, complex(1,1))
 
     def test_reciprocal(self):
-        from numpypy import array, reciprocal, complex64, complex128, clongdouble
+        from numpypy import array, reciprocal
         inf = float('inf')
         nan = float('nan')
         #complex
@@ -209,7 +209,7 @@
                     complex(-r, i),
                     -0j, 0j, cnan,
                     cnan, cnan, cnan]
-        for c, rel_err in ((complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)):
+        for c, rel_err in (('complex64', 2e-7), ('complex128', 2e-15), ('clongdouble', 2e-15)):
             actual = reciprocal(array([orig], dtype=c))
             for b, a, e in zip(orig, actual, expected):
                 assert (a[0].real - e.real) < rel_err
@@ -223,18 +223,19 @@
         raises(TypeError, trunc, a)
 
     def test_copysign(self):
-        from numpypy import copysign, complex128
+        from numpypy import copysign, dtype
+        complex128 = dtype('complex128').type
         a = complex128(complex(-5., 5.))
         b = complex128(complex(0., 0.))
         raises(TypeError, copysign, a, b)
 
     def test_exp2(self):
-        from numpypy import array, exp2, complex128, complex64, clongdouble
+        from numpypy import array, exp2
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
         cmpl = complex
-        for c, rel_err in ((complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)):
+        for c, rel_err in (('complex64', 2e-7), ('complex128', 2e-15), ('clongdouble', 2e-15)):
             a = [cmpl(-5., 0), cmpl(-5., -5.), cmpl(-5., 5.),
                        cmpl(0., -5.), cmpl(0., 0.), cmpl(0., 5.),
                        cmpl(-0., -5.), cmpl(-0., 0.), cmpl(-0., 5.),
@@ -265,12 +266,12 @@
 
     def test_expm1(self):
         import math, cmath
-        from numpypy import array, expm1, complex128, complex64, clongdouble
+        from numpypy import array, expm1
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
         cmpl = complex
-        for c, rel_err in ((complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)):
+        for c, rel_err in (('complex64', 2e-7), ('complex128', 2e-15), ('clongdouble', 2e-15)):
             a = [cmpl(-5., 0), cmpl(-5., -5.), cmpl(-5., 5.),
                        cmpl(0., -5.), cmpl(0., 0.), cmpl(0., 5.),
                        cmpl(-0., -5.), cmpl(-0., 0.), cmpl(-0., 5.),
@@ -305,14 +306,12 @@
 
     def test_not_complex(self):
         from numpypy import (radians, deg2rad, degrees, rad2deg,
-                  isneginf, isposinf, logaddexp, logaddexp2, fmod,
+                  logaddexp, logaddexp2, fmod,
                   arctan2)
         raises(TypeError, radians, complex(90,90))
         raises(TypeError, deg2rad, complex(90,90))
         raises(TypeError, degrees, complex(90,90))
         raises(TypeError, rad2deg, complex(90,90))
-        raises(TypeError, isneginf, complex(1, 1))
-        raises(TypeError, isposinf, complex(1, 1))
         raises(TypeError, logaddexp, complex(1, 1), complex(3, 3))
         raises(TypeError, logaddexp2, complex(1, 1), complex(3, 3))
         raises(TypeError, arctan2, complex(1, 1), complex(3, 3))
@@ -341,12 +340,12 @@
         nan = float('nan')
         cmpl = complex
         from math import copysign
-        from numpypy import power, array, complex128, complex64
+        from numpypy import power, array
         # note: in some settings (namely a x86-32 build without the JIT),
         # gcc optimizes the code in rlib.rcomplex.c_pow() to not truncate
         # the 10-byte values down to 8-byte values.  It ends up with more
         # imprecision than usual (hence 2e-13 instead of 2e-15).
-        for c,rel_err in ((complex128, 2e-13), (complex64, 4e-7)):
+        for c,rel_err in (('complex128', 2e-13), ('complex64', 4e-7)):
             a = array([cmpl(-5., 0), cmpl(-5., -5.), cmpl(-5., 5.),
                        cmpl(0., -5.), cmpl(0., 0.), cmpl(0., 5.),
                        cmpl(-0., -5.), cmpl(-0., 0.), cmpl(-0., 5.),
@@ -378,11 +377,11 @@
                     self.rAlmostEqual(t1, t2, rel_err=rel_err, msg=msg)
 
     def test_conjugate(self):
-        from numpypy import conj, conjugate, complex128, complex64
+        from numpypy import conj, conjugate, dtype
         import numpypy as np
 
-        c0 = complex128(complex(2.5, 0))
-        c1 = complex64(complex(1, 2))
+        c0 = dtype('complex128').type(complex(2.5, 0))
+        c1 = dtype('complex64').type(complex(1, 2))
 
         assert conj is conjugate
         assert conj(c0) == c0
@@ -395,7 +394,8 @@
 
         assert np.conjugate(1+2j) == 1-2j
 
-        x = np.eye(2) + 1j * np.eye(2)
+        eye2 = np.array([[1, 0], [0, 1]])
+        x = eye2 + 1j * eye2
         for a, b in zip(np.conjugate(x), np.array([[ 1.-1.j,  0.-0.j], [ 0.-0.j,  1.-1.j]])):
             assert a[0] == b[0]
             assert a[1] == b[1]
@@ -403,7 +403,7 @@
     def test_logn(self):
         import math, cmath
         # log and log10 are tested in math (1:1 from rcomplex)
-        from numpypy import log2, array, complex128, complex64, log1p
+        from numpypy import log2, array, log1p
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
@@ -418,7 +418,7 @@
              cmpl(ninf, ninf), cmpl(5., inf), cmpl(5., ninf),
              cmpl(nan, 5.), cmpl(5., nan), cmpl(nan, nan),
             ]
-        for c,rel_err in ((complex128, 2e-15), (complex64, 1e-7)):
+        for c,rel_err in (('complex128', 2e-15), ('complex64', 1e-7)):
             b = log2(array(a,dtype=c))
             for i in range(len(a)):
                 try:
@@ -438,7 +438,7 @@
                 t1 = float(res.imag)
                 t2 = float(b[i].imag)
                 self.rAlmostEqual(t1, t2, rel_err=rel_err, msg=msg)
-        for c,rel_err in ((complex128, 2e-15), (complex64, 1e-7)):
+        for c,rel_err in (('complex128', 2e-15), ('complex64', 1e-7)):
             b = log1p(array(a,dtype=c))
             for i in range(len(a)):
                 try:
@@ -492,9 +492,9 @@
             assert c[i] == max(a[i], b[i])
 
     def test_basic(self):
-        from numpypy import (complex128, complex64, add, array, dtype,
+        from numpypy import (dtype, add, array, dtype,
             subtract as sub, multiply, divide, negative, absolute as abs,
-            floor_divide, real, imag, sign, clongdouble)
+            floor_divide, real, imag, sign)
         from numpypy import (equal, not_equal, greater, greater_equal, less,
                 less_equal, isnan)
         assert real(4.0) == 4.0
@@ -525,7 +525,8 @@
         assert str(a.real) == 'abc'
         # numpy imag for flexible types returns self
         assert str(a.imag) == 'abc'
-        for complex_ in complex64, complex128, clongdouble:
+        for t in 'complex64', 'complex128', 'clongdouble':
+            complex_ = dtype(t).type
             O = complex(0, 0)
             c0 = complex_(complex(2.5, 0))
             c1 = complex_(complex(1, 2))
@@ -614,10 +615,11 @@
         import numpypy as np
         rAlmostEqual = self.rAlmostEqual
 
-        for complex_, testcases in (
-                 (np.complex128, self.testcases128),
-                 #(np.complex64, self.testcases64),
+        for t, testcases in (
+                 ('complex128', self.testcases128),
+                 #('complex64', self.testcases64),
                 ):
+            complex_ = np.dtype(t).type
             for id, fn, ar, ai, er, ei, flags in testcases:
                 arg = complex_(complex(ar, ai))
                 expected = (er, ei)
@@ -665,6 +667,6 @@
             sys.stderr.write('\n')
 
     def test_complexbox_to_pycomplex(self):
-        from numpypy import complex128
-        x = complex128(3.4j)
+        from numpypy import dtype
+        x = dtype('complex128').type(3.4j)
         assert complex(x) == 3.4j
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -90,8 +90,9 @@
         # test doubles and complex doubles as the logic is the same.
 
         # check doubles
-        from numpypy import array, nan, zeros, complex128, arange
+        from numpypy import array, zeros, arange
         from math import isnan
+        nan = float('nan')
         a = array([nan, 1, 0])
         b = a.copy()
         b.sort()
@@ -102,7 +103,7 @@
         assert (b == [2, 1, 0]).all()
 
         # check complex
-        a = zeros(9, dtype=complex128)
+        a = zeros(9, dtype='complex128')
         a.real += [nan, nan, nan, 1, 0, 1, 1, 0, 0]
         a.imag += [nan, 1, 0, nan, nan, 1, 0, 1, 0]
         b = a.copy()
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -239,7 +239,7 @@
         assert math.isnan(true_divide(0, 0))
 
     def test_fabs(self):
-        from numpypy import array, fabs, complex128
+        from numpypy import array, fabs
         from math import fabs as math_fabs, isnan
 
         a = array([-5.0, -0.0, 1.0])
@@ -332,7 +332,7 @@
         assert(multiply.reduce(a) == array([0, 3640, 12320])).all()
 
     def test_rint(self):
-        from numpypy import array, complex, rint, isnan
+        from numpypy import array, dtype, rint, isnan
         import sys
 
         nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
@@ -384,7 +384,7 @@
             [False, True, True]).all()
 
     def test_reciprocal(self):
-        from numpypy import array, reciprocal, complex64, complex128
+        from numpypy import array, reciprocal
 
         inf = float('inf')
         nan = float('nan')
@@ -417,7 +417,7 @@
             assert c[i] == a[i] - b[i]
 
     def test_floorceiltrunc(self):
-        from numpypy import array, floor, ceil, trunc, complex128
+        from numpypy import array, floor, ceil, trunc
         import math
         ninf, inf = float("-inf"), float("inf")
         a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
@@ -782,10 +782,11 @@
         assert invert(False) == True
 
     def test_shift(self):
-        from numpypy import left_shift, right_shift, bool
+        from numpypy import left_shift, right_shift, dtype
 
         assert (left_shift([5, 1], [2, 13]) == [20, 2**13]).all()
         assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
+        bool_ = dtype('bool').type
         assert left_shift(bool(1), 3) == left_shift(1, 3)
         assert right_shift(bool(1), 3) == right_shift(1, 3)
 
@@ -832,11 +833,11 @@
         assert (true_divide(arange(3), array([2, 2, 2])) == array([0, 0.5, 1])).all()
 
     def test_isnan_isinf(self):
-        from numpypy import isnan, isinf, float64, array
+        from numpypy import isnan, isinf, array, dtype
         assert isnan(float('nan'))
         assert not isnan(3)
         assert not isinf(3)
-        assert isnan(float64(float('nan')))
+        assert isnan(dtype('float64').type(float('nan')))
         assert not isnan(3)
         assert isinf(float('inf'))
         assert not isnan(3.5)
@@ -847,39 +848,6 @@
         assert (isinf(array([0.2, float('inf'), float('nan')])) == [False, True, False]).all()
         assert isinf(array([0.2])).dtype.kind == 'b'
 
-    def test_isposinf_isneginf(self):
-        from numpypy import isneginf, isposinf
-        assert isposinf(float('inf'))
-        assert not isposinf(3)
-        assert not isneginf(3)
-        assert not isposinf(float('-inf'))
-        assert not isposinf(float('nan'))
-        assert not isposinf(0)
-        assert not isposinf(0.0)
-        assert isneginf(float('-inf'))
-        assert not isneginf(float('inf'))
-        assert not isneginf(float('nan'))
-        assert not isneginf(0)
-        assert not isneginf(0.0)
-
-    def test_isfinite(self):
-        from numpypy import isfinite
-        inf = float('inf')
-        ninf = -float('inf')
-        nan = float('nan')
-        assert (isfinite([0, 0.0, 1e50, -1e-50]) ==
-            [True, True, True, True]).all()
-        assert (isfinite([ninf, inf, -nan, nan]) ==
-            [False, False, False, False]).all()
-        assert (isfinite([1, 2, 3]) == [True, True, True]).all()
-
-        a = [complex(0, 0), complex(1e50, -1e-50), complex(inf, 0),
-             complex(inf, inf), complex(inf, ninf), complex(0, inf),
-             complex(ninf, ninf), complex(nan, 0), complex(0, nan),
-             complex(nan, nan)]
-        assert (isfinite(a) == [True, True, False, False, False,
-                        False, False, False, False, False]).all()
-
     def test_logical_ops(self):
         from numpypy import logical_and, logical_or, logical_xor, logical_not
 

From noreply at buildbot.pypy.org  Tue Oct 29 22:34:38 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 22:34:38 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: these functions exist in core
	numpy now
Message-ID: <20131029213438.E610E1C1052@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67708:497092f845b3
Date: 2013-10-29 17:12 -0400
http://bitbucket.org/pypy/pypy/changeset/497092f845b3/

Log:	these functions exist in core numpy now

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1228,24 +1228,6 @@
         assert d[0] == 110
         assert d[1] == 12
 
-    def test_mean(self):
-        from numpypy import array, arange
-        a = array(range(5))
-        assert a.mean() == 2.0
-        assert a[:4].mean() == 1.5
-        a = array(range(105)).reshape(3, 5, 7)
-        b = a.mean(axis=0)
-        assert b[0, 0] == 35.
-        assert a.mean(axis=0)[0, 0] == 35
-        assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
-        assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
-        assert (arange(10).reshape(5, 2).mean(axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all()
-        assert (a.mean(axis=-1) == a.mean(axis=2)).all()
-        raises(IndexError, a.mean, -4)
-        raises(IndexError, a.mean, 3)
-        a = arange(10).reshape(5, 2)
-        assert (a.mean(1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all()
-
     def test_sum(self):
         from numpypy import array, zeros
         a = array(range(5))
@@ -1571,17 +1553,6 @@
         assert a[:, 0].tolist() == [17.1, 40.3]
         assert a[0].tolist() == [17.1, 27.2]
 
-    def test_var(self):
-        from numpypy import array, arange
-        a = array(range(10))
-        assert a.var() == 8.25
-        a = array([5.0])
-        assert a.var() == 0.0
-        a = arange(10).reshape(5, 2)
-        assert a.var() == 8.25
-        assert (a.var(0) == [8, 8]).all()
-        assert (a.var(1) == [.25] * 5).all()
-
     def test_concatenate(self):
         from numpypy import array, concatenate, dtype
         a1 = array([0,1,2])
@@ -1663,13 +1634,6 @@
                                             dtype=[('x', int), ('y', float)])))
         assert str(exc.value).startswith('invalid type promotion')
 
-    def test_std(self):
-        from numpypy import array
-        a = array(range(10))
-        assert a.std() == 2.8722813232690143
-        a = array([5.0])
-        assert a.std() == 0.0
-
     def test_flatten(self):
         from numpypy import array
 

From noreply at buildbot.pypy.org  Tue Oct 29 22:34:37 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 22:34:37 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: some more fixes
Message-ID: <20131029213437.9FC971C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67707:92a9f1d35d4a
Date: 2013-10-29 17:09 -0400
http://bitbucket.org/pypy/pypy/changeset/92a9f1d35d4a/

Log:	some more fixes

diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py
--- a/pypy/module/micronumpy/appbridge.py
+++ b/pypy/module/micronumpy/appbridge.py
@@ -12,8 +12,8 @@
         self.w_import = space.appexec([], """():
         def f():
             import sys
-            __import__('numpypy.core._methods')
-            return sys.modules['numpypy.core._methods']
+            __import__('numpy.core._methods')
+            return sys.modules['numpy.core._methods']
         return f
         """)
 
diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -1,4 +1,5 @@
 from _numpypy.multiarray import *
 from _numpypy.umath import *
 
+newaxis = None
 ufunc = type(sin)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -583,8 +583,7 @@
 
     def test_newaxis(self):
         import math
-        from numpypy import array, cos, zeros
-        from numpypy.core.numeric import newaxis
+        from numpypy import array, cos, zeros, newaxis
         a = array(range(5))
         b = array([range(5)])
         assert (a[newaxis] == b).all()
@@ -597,8 +596,7 @@
         assert ((cos(a)[:,newaxis] * cos(b).T) == expected).all()
 
     def test_newaxis_slice(self):
-        from numpypy import array
-        from numpypy.core.numeric import newaxis
+        from numpypy import array, newaxis
 
         a = array(range(5))
         b = array(range(1,5))
@@ -610,16 +608,14 @@
         assert (a[newaxis,1:] == c).all()
 
     def test_newaxis_assign(self):
-        from numpypy import array
-        from numpypy.core.numeric import newaxis
+        from numpypy import array, newaxis
 
         a = array(range(5))
         a[newaxis,1] = [2]
         assert a[1] == 2
 
     def test_newaxis_virtual(self):
-        from numpypy import array
-        from numpypy.core.numeric import newaxis
+        from numpypy import array, newaxis
 
         a = array(range(5))
         b = (a + a)[newaxis]
@@ -627,16 +623,14 @@
         assert (b == c).all()
 
     def test_newaxis_then_slice(self):
-        from numpypy import array
-        from numpypy.core.numeric import newaxis
+        from numpypy import array, newaxis
         a = array(range(5))
         b = a[newaxis]
         assert b.shape == (1, 5)
         assert (b[0,1:] == a[1:]).all()
 
     def test_slice_then_newaxis(self):
-        from numpypy import array
-        from numpypy.core.numeric import newaxis
+        from numpypy import array, newaxis
         a = array(range(5))
         b = a[2:]
         assert (b[newaxis] == [[2, 3, 4]]).all()

From noreply at buildbot.pypy.org  Tue Oct 29 22:34:40 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 22:34:40 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: flexible_repr lives in numpy core
Message-ID: <20131029213440.323081C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67709:e4361c7a16d5
Date: 2013-10-29 17:20 -0400
http://bitbucket.org/pypy/pypy/changeset/e4361c7a16d5/

Log:	flexible_repr lives in numpy core

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2706,6 +2706,8 @@
         from numpypy import array
         assert repr(array([1, 2, 3])) == 'array([1, 2, 3])'
         assert str(array([1, 2, 3])) == 'array([1, 2, 3])'
+        assert repr(array(['abc'], 'S3')) == "array(['abc'])"
+        assert str(array(['abc'], 'S3')) == "array(['abc'])"
 
     def teardown_class(cls):
         if option.runappdirect:
@@ -2849,19 +2851,6 @@
         assert str(a.dtype) == '|S1'
         assert a == 'x'
 
-    def test_flexible_repr(self):
-        from numpypy import array
-        a = array(['abc'],'S3')
-        s = repr(a)
-        # simplify test for \n in repr
-        assert s.replace('\n', '') == "array(['abc'],       dtype='|S3')"
-        # but make sure it exists
-        assert s.find('\n') == 15
-        a = array(['abc','defg','ab'])
-        s = repr(a)
-        assert s.replace('\n', '') == \
-                      "array(['abc', 'defg', 'ab'],       dtype='|S4')"
-
     def test_pickle(self):
         from numpypy import dtype, array
         from cPickle import loads, dumps

From noreply at buildbot.pypy.org  Tue Oct 29 22:34:41 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 22:34:41 +0100 (CET)
Subject: [pypy-commit] pypy default: these tests are pypy-specific
Message-ID: <20131029213441.98E991C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67710:177c957bd282
Date: 2013-10-29 17:21 -0400
http://bitbucket.org/pypy/pypy/changeset/177c957bd282/

Log:	these tests are pypy-specific

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1350,7 +1350,9 @@
         assert a.argmax() == 5
         assert a.argmax(axis=None, out=None) == 5
         assert a[:2, ].argmax() == 3
-        raises(NotImplementedError, a.argmax, axis=0)
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.argmax, axis=0)
 
     def test_argmin(self):
         from numpypy import array
@@ -1359,7 +1361,9 @@
         assert a.argmin(axis=None, out=None) == 3
         b = array([])
         raises(ValueError, "b.argmin()")
-        raises(NotImplementedError, a.argmin, axis=0)
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.argmin, axis=0)
 
     def test_all(self):
         from numpypy import array
@@ -2214,7 +2218,9 @@
         b = a.T
         assert(b[:, 0] == a[0, :]).all()
         assert (a.transpose() == b).all()
-        raises(NotImplementedError, a.transpose, (1, 0, 2))
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.transpose, (1, 0, 2))
 
     def test_flatiter(self):
         from numpypy import array, flatiter, arange, zeros
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -18,7 +18,9 @@
 
             a = arange(100, dtype=dtype)
             assert (a.argsort() == a).all()
-        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
 
     def test_argsort_ndim(self):
         from numpypy import array
@@ -76,8 +78,10 @@
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
             b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
             c = a.copy()
-            exc = raises(NotImplementedError, a.sort)
-            assert exc.value[0].find('supported') >= 0
+            import sys
+            if '__pypy__' in sys.builtin_module_names:
+                exc = raises(NotImplementedError, a.sort)
+                assert exc.value[0].find('supported') >= 0
             #assert (a == b).all(), \
             #    'a,orig,dtype %r,%r,%r' % (a,c,dtype)
 

From noreply at buildbot.pypy.org  Tue Oct 29 22:34:42 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 22:34:42 +0100 (CET)
Subject: [pypy-commit] pypy default: forgotten hg add
Message-ID: <20131029213442.BF6A81C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67711:cc6ab6eafe52
Date: 2013-10-29 17:33 -0400
http://bitbucket.org/pypy/pypy/changeset/cc6ab6eafe52/

Log:	forgotten hg add

diff --git a/pypy/module/micronumpy/conversion_utils.py b/pypy/module/micronumpy/conversion_utils.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/conversion_utils.py
@@ -0,0 +1,20 @@
+from pypy.interpreter.error import OperationError
+from pypy.module.micronumpy.constants import *
+
+def clipmode_converter(space, w_mode):
+    if space.is_none(w_mode):
+        return NPY_RAISE
+    if space.isinstance_w(w_mode, space.w_str):
+        mode = space.str_w(w_mode)
+        if mode.startswith('C') or mode.startswith('c'):
+            return NPY_CLIP
+        if mode.startswith('W') or mode.startswith('w'):
+            return NPY_WRAP
+        if mode.startswith('R') or mode.startswith('r'):
+            return NPY_RAISE
+    elif space.isinstance_w(w_mode, space.w_int):
+        mode = space.int_w(w_mode)
+        if NPY_CLIP <= mode <= NPY_RAISE:
+            return mode
+    raise OperationError(space.w_TypeError,
+                         space.wrap("clipmode not understood"))

From noreply at buildbot.pypy.org  Tue Oct 29 22:34:46 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 22:34:46 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: merge default
Message-ID: <20131029213446.AFDFB1C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67712:284aaabac579
Date: 2013-10-29 17:33 -0400
http://bitbucket.org/pypy/pypy/changeset/284aaabac579/

Log:	merge default

diff too long, truncating to 2000 out of 3032 lines

diff --git a/pypy/module/micronumpy/conversion_utils.py b/pypy/module/micronumpy/conversion_utils.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/conversion_utils.py
@@ -0,0 +1,20 @@
+from pypy.interpreter.error import OperationError
+from pypy.module.micronumpy.constants import *
+
+def clipmode_converter(space, w_mode):
+    if space.is_none(w_mode):
+        return NPY_RAISE
+    if space.isinstance_w(w_mode, space.w_str):
+        mode = space.str_w(w_mode)
+        if mode.startswith('C') or mode.startswith('c'):
+            return NPY_CLIP
+        if mode.startswith('W') or mode.startswith('w'):
+            return NPY_WRAP
+        if mode.startswith('R') or mode.startswith('r'):
+            return NPY_RAISE
+    elif space.isinstance_w(w_mode, space.w_int):
+        mode = space.int_w(w_mode)
+        if NPY_CLIP <= mode <= NPY_RAISE:
+            return mode
+    raise OperationError(space.w_TypeError,
+                         space.wrap("clipmode not understood"))
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1327,7 +1327,9 @@
         assert a.argmax() == 5
         assert a.argmax(axis=None, out=None) == 5
         assert a[:2, ].argmax() == 3
-        raises(NotImplementedError, a.argmax, axis=0)
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.argmax, axis=0)
 
     def test_argmin(self):
         from numpypy import array
@@ -1336,7 +1338,9 @@
         assert a.argmin(axis=None, out=None) == 3
         b = array([])
         raises(ValueError, "b.argmin()")
-        raises(NotImplementedError, a.argmin, axis=0)
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.argmin, axis=0)
 
     def test_all(self):
         from numpypy import array
@@ -2165,7 +2169,9 @@
         b = a.T
         assert(b[:, 0] == a[0, :]).all()
         assert (a.transpose() == b).all()
-        raises(NotImplementedError, a.transpose, (1, 0, 2))
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.transpose, (1, 0, 2))
 
     def test_flatiter(self):
         from numpypy import array, flatiter, arange, zeros
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -18,7 +18,9 @@
 
             a = arange(100, dtype=dtype)
             assert (a.argsort() == a).all()
-        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
 
     def test_argsort_ndim(self):
         from numpypy import array
@@ -76,8 +78,10 @@
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
             b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
             c = a.copy()
-            exc = raises(NotImplementedError, a.sort)
-            assert exc.value[0].find('supported') >= 0
+            import sys
+            if '__pypy__' in sys.builtin_module_names:
+                exc = raises(NotImplementedError, a.sort)
+                assert exc.value[0].find('supported') >= 0
             #assert (a == b).all(), \
             #    'a,orig,dtype %r,%r,%r' % (a,c,dtype)
 
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -6,7 +6,8 @@
 import operator
 from rpython.tool.pairtype import pair, pairtype
 from rpython.annotator.model import SomeObject, SomeInteger, SomeBool, s_Bool
-from rpython.annotator.model import SomeString, SomeChar, SomeList, SomeDict
+from rpython.annotator.model import SomeString, SomeChar, SomeList, SomeDict,\
+     SomeOrderedDict
 from rpython.annotator.model import SomeUnicodeCodePoint, SomeUnicodeString
 from rpython.annotator.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue
 from rpython.annotator.model import SomeInstance, SomeBuiltin, SomeIterator
@@ -581,7 +582,8 @@
 class __extend__(pairtype(SomeDict, SomeDict)):
 
     def union((dic1, dic2)):
-        return SomeDict(dic1.dictdef.union(dic2.dictdef))
+        assert dic1.__class__ == dic2.__class__
+        return dic1.__class__(dic1.dictdef.union(dic2.dictdef))
 
 
 class __extend__(pairtype(SomeDict, SomeObject)):
@@ -840,6 +842,7 @@
 _make_none_union('SomeString',      'no_nul=obj.no_nul, can_be_None=True')
 _make_none_union('SomeUnicodeString', 'can_be_None=True')
 _make_none_union('SomeList',         'obj.listdef')
+_make_none_union('SomeOrderedDict',          'obj.dictdef')
 _make_none_union('SomeDict',          'obj.dictdef')
 _make_none_union('SomeWeakRef',         'obj.classdef')
 
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -5,9 +5,10 @@
 from __future__ import absolute_import
 
 import sys, types, inspect, weakref
+from collections import OrderedDict
 
 from rpython.flowspace.model import Constant
-from rpython.annotator.model import (
+from rpython.annotator.model import (SomeOrderedDict,
     SomeString, SomeChar, SomeFloat, SomePtr, unionof, SomeInstance, SomeDict,
     SomeBuiltin, SomePBC, SomeInteger, TLS, SomeAddress, SomeUnicodeCodePoint,
     s_None, s_ImpossibleValue, SomeLLADTMeth, SomeBool, SomeTuple,
@@ -370,7 +371,7 @@
                 for e in x:
                     listdef.generalize(self.immutablevalue(e, False))
                 result = SomeList(listdef)
-        elif tp is dict or tp is r_dict:
+        elif tp is dict or tp is r_dict or tp is OrderedDict:
             if need_const:
                 key = Constant(x)
                 try:
@@ -412,7 +413,10 @@
                     dictdef.generalize_key(self.immutablevalue(ek, False))
                     dictdef.generalize_value(self.immutablevalue(ev, False))
                     dictdef.seen_prebuilt_key(ek)
-                result = SomeDict(dictdef)
+                if tp is OrderedDict:
+                    result = SomeOrderedDict(dictdef)
+                else:
+                    result = SomeDict(dictdef)
         elif tp is weakref.ReferenceType:
             x1 = x()
             if x1 is None:
diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -2,11 +2,13 @@
 Built-in functions.
 """
 import sys
+from collections import OrderedDict
 
 from rpython.annotator.model import (
     SomeInteger, SomeObject, SomeChar, SomeBool, SomeString, SomeTuple, s_Bool,
     SomeUnicodeCodePoint, SomeAddress, SomeFloat, unionof, SomeUnicodeString,
     SomePBC, SomeInstance, SomeDict, SomeList, SomeWeakRef, SomeIterator,
+    SomeOrderedDict,
     SomeByteArray, annotation_to_lltype, lltype_to_annotation,
     ll_to_annotation, add_knowntypedata, s_ImpossibleValue,)
 from rpython.annotator.bookkeeper import getbookkeeper
@@ -298,6 +300,10 @@
     dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn)
     return SomeDict(dictdef)
 
+def robjmodel_r_ordereddict(s_eqfn, s_hashfn):
+    dictdef = getbookkeeper().getdictdef(is_r_dict=True)
+    dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn)
+    return SomeOrderedDict(dictdef)
 
 def robjmodel_hlinvoke(s_repr, s_llcallable, *args_s):
     from rpython.rtyper import rmodel
@@ -357,6 +363,8 @@
 BUILTIN_ANALYZERS[rpython.rlib.rarithmetic.longlongmask] = rarith_longlongmask
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.instantiate] = robjmodel_instantiate
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_dict] = robjmodel_r_dict
+BUILTIN_ANALYZERS[rpython.rlib.objectmodel.r_ordereddict] = robjmodel_r_ordereddict
+BUILTIN_ANALYZERS[OrderedDict] = lambda : SomeOrderedDict(getbookkeeper().getdictdef())
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.hlinvoke] = robjmodel_hlinvoke
 BUILTIN_ANALYZERS[rpython.rlib.objectmodel.keepalive_until_here] = robjmodel_keepalive_until_here
 BUILTIN_ANALYZERS[rpython.rtyper.lltypesystem.llmemory.cast_ptr_to_adr] = llmemory_cast_ptr_to_adr
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -32,6 +32,7 @@
 import inspect
 import weakref
 from types import BuiltinFunctionType, MethodType
+from collections import OrderedDict
 
 import rpython
 from rpython.tool import descriptor
@@ -355,6 +356,18 @@
         else:
             return '{...%s...}' % (len(const),)
 
+class SomeOrderedDict(SomeDict):
+    knowntype = OrderedDict
+
+    def method_copy(dct):
+        return SomeOrderedDict(dct.dictdef)
+
+    def method_update(dct1, dct2):
+        if s_None.contains(dct2):
+            return SomeImpossibleValue()
+        assert isinstance(dct2, SomeOrderedDict), "OrderedDict.update(dict) not allowed"
+        dct1.dictdef.union(dct2.dictdef)
+
 
 class SomeIterator(SomeObject):
     "Stands for an iterator returning objects from a given container."
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -628,8 +628,11 @@
     The functions key_eq() and key_hash() are used by the key comparison
     algorithm."""
 
+    def _newdict(self):
+        return {}
+
     def __init__(self, key_eq, key_hash, force_non_null=False):
-        self._dict = {}
+        self._dict = self._newdict()
         self.key_eq = key_eq
         self.key_hash = key_hash
         self.force_non_null = force_non_null
@@ -664,7 +667,7 @@
         return dk.key, value
 
     def copy(self):
-        result = r_dict(self.key_eq, self.key_hash)
+        result = self.__class__(self.key_eq, self.key_hash)
         result.update(self)
         return result
 
@@ -700,6 +703,11 @@
     def __hash__(self):
         raise TypeError("cannot hash r_dict instances")
 
+class r_ordereddict(r_dict):
+    def _newdict(self):
+        from collections import OrderedDict
+        
+        return OrderedDict()
 
 class _r_dictkey(object):
     __slots__ = ['dic', 'key', 'hash']
@@ -735,7 +743,7 @@
     Function and staticmethod objects are duplicated, which means
     that annotation will not consider them as identical to another
     copy in another unrelated class.
-    
+
     By default, "special" methods and class attributes, with a name
     like "__xxx__", are not copied unless they are "__init__" or
     "__del__".  The list can be changed with the optional second
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -522,8 +522,10 @@
     A = lltype.typeOf(source)
     assert A == lltype.typeOf(dest)
     if isinstance(A.TO, lltype.GcArray):
-        assert isinstance(A.TO.OF, lltype.Ptr)
-        assert A.TO.OF.TO._gckind == 'gc'
+        if isinstance(A.TO.OF, lltype.Ptr):
+            assert A.TO.OF.TO._gckind == 'gc'
+        else:
+            assert isinstance(A.TO.OF, lltype.Struct)
     else:
         assert isinstance(A.TO, lltype.GcStruct)
         assert A.TO._arrayfld is not None
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -0,0 +1,1149 @@
+import sys
+from rpython.tool.pairtype import pairtype
+from rpython.flowspace.model import Constant
+from rpython.rtyper.rdict import AbstractDictRepr, AbstractDictIteratorRepr
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rlib import objectmodel, jit, rgc
+from rpython.rlib.debug import ll_assert
+from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rtyper import rmodel
+from rpython.rtyper.error import TyperError
+from rpython.rtyper.annlowlevel import llhelper
+
+
+# ____________________________________________________________
+#
+#  generic implementation of RPython dictionary, with parametric DICTKEY and
+#  DICTVALUE types. The basic implementation is a sparse array of indexes
+#  plus a dense array of structs that contain keys and values. struct looks
+#  like that:
+#
+#
+#    struct dictentry {
+#        DICTKEY key;
+#        DICTVALUE value;
+#        long f_hash;        # (optional) key hash, if hard to recompute
+#        bool f_valid;      # (optional) the entry is filled
+#    }
+#
+#    struct dicttable {
+#        int num_items;
+#        int num_used_items;
+#        int resize_counter;
+#        {byte, short, int, long} *indexes;
+#        dictentry *entries;
+#        lookup_function_no; # one of the four possible functions for different
+#                         # size dicts
+#        (Function DICTKEY, DICTKEY -> bool) *fnkeyeq;
+#        (Function DICTKEY -> int) *fnkeyhash;
+#    }
+#
+#
+
+def ll_call_lookup_function(d, key, hash, flag):
+    DICT = lltype.typeOf(d).TO
+    fun = d.lookup_function_no
+    if fun == FUNC_BYTE:
+        return DICT.lookup_family.byte_lookup_function(d, key, hash, flag)
+    elif fun == FUNC_SHORT:
+        return DICT.lookup_family.short_lookup_function(d, key, hash, flag)
+    elif IS_64BIT and fun == FUNC_INT:
+        return DICT.lookup_family.int_lookup_function(d, key, hash, flag)
+    elif fun == FUNC_LONG:
+        return DICT.lookup_family.long_lookup_function(d, key, hash, flag)
+    assert False
+
+def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None,
+                ll_fasthash_function=None, ll_hash_function=None,
+                ll_eq_function=None, method_cache={},
+                dummykeyobj=None, dummyvalueobj=None, rtyper=None,
+                setup_lookup_funcs=True):
+    # get the actual DICT type. if DICT is None, it's created, otherwise
+    # forward reference is becoming DICT
+    if DICT is None:
+        DICT = lltype.GcForwardReference()
+    # compute the shape of the DICTENTRY structure
+    entryfields = []
+    entrymeths = {
+        'allocate': lltype.typeMethod(_ll_malloc_entries),
+        'delete': _ll_free_entries,
+        'must_clear_key':   (isinstance(DICTKEY, lltype.Ptr)
+                             and DICTKEY._needsgc()),
+        'must_clear_value': (isinstance(DICTVALUE, lltype.Ptr)
+                             and DICTVALUE._needsgc()),
+        }
+
+    # * the key
+    entryfields.append(("key", DICTKEY))
+
+    # * the state of the entry - trying to encode it as dummy objects
+    if dummykeyobj:
+        # all the state can be encoded in the key
+        entrymeths['dummy_obj'] = dummykeyobj
+        entrymeths['valid'] = ll_valid_from_key
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_key
+        # the key is overwritten by 'dummy' when the entry is deleted
+        entrymeths['must_clear_key'] = False
+
+    elif dummyvalueobj:
+        # all the state can be encoded in the value
+        entrymeths['dummy_obj'] = dummyvalueobj
+        entrymeths['valid'] = ll_valid_from_value
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_value
+        # value is overwritten by 'dummy' when entry is deleted
+        entrymeths['must_clear_value'] = False
+
+    else:
+        # we need a flag to know if the entry was ever used
+        entryfields.append(("f_valid", lltype.Bool))
+        entrymeths['valid'] = ll_valid_from_flag
+        entrymeths['mark_deleted'] = ll_mark_deleted_in_flag
+
+    # * the value
+    entryfields.append(("value", DICTVALUE))
+
+    if ll_fasthash_function is None:
+        entryfields.append(("f_hash", lltype.Signed))
+        entrymeths['hash'] = ll_hash_from_cache
+    else:
+        entrymeths['hash'] = ll_hash_recomputed
+        entrymeths['fasthashfn'] = ll_fasthash_function
+
+    # Build the lltype data structures
+    DICTENTRY = lltype.Struct("dictentry", *entryfields)
+    DICTENTRYARRAY = lltype.GcArray(DICTENTRY,
+                                    adtmeths=entrymeths)
+    fields =          [ ("num_items", lltype.Signed),
+                        ("num_used_items", lltype.Signed),
+                        ("resize_counter", lltype.Signed),
+                        ("indexes", llmemory.GCREF),
+                        ("lookup_function_no", lltype.Signed),
+                        ("entries", lltype.Ptr(DICTENTRYARRAY)) ]
+    if get_custom_eq_hash is not None:
+        r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash()
+        fields.extend([ ("fnkeyeq", r_rdict_eqfn.lowleveltype),
+                        ("fnkeyhash", r_rdict_hashfn.lowleveltype) ])
+        adtmeths = {
+            'keyhash':        ll_keyhash_custom,
+            'keyeq':          ll_keyeq_custom,
+            'r_rdict_eqfn':   r_rdict_eqfn,
+            'r_rdict_hashfn': r_rdict_hashfn,
+            'paranoia':       True,
+            }
+    else:
+        # figure out which functions must be used to hash and compare
+        ll_keyhash = ll_hash_function
+        ll_keyeq = ll_eq_function
+        ll_keyhash = lltype.staticAdtMethod(ll_keyhash)
+        if ll_keyeq is not None:
+            ll_keyeq = lltype.staticAdtMethod(ll_keyeq)
+        adtmeths = {
+            'keyhash':  ll_keyhash,
+            'keyeq':    ll_keyeq,
+            'paranoia': False,
+            }
+    adtmeths['KEY']   = DICTKEY
+    adtmeths['VALUE'] = DICTVALUE
+    adtmeths['lookup_function'] = lltype.staticAdtMethod(ll_call_lookup_function)
+    adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict)
+
+    family = LookupFamily()
+    adtmeths['lookup_family'] = family
+
+    DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths,
+                                *fields))
+
+    family.empty_array = DICTENTRYARRAY.allocate(0)
+    if setup_lookup_funcs:
+        _setup_lookup_funcs(DICT, rtyper, family)
+    return DICT
+
+def _setup_lookup_funcs(DICT, rtyper, family):
+    DICTKEY = DICT.entries.TO.OF.key
+    LOOKUP_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT), DICTKEY,
+                                              lltype.Signed, lltype.Signed],
+                                              lltype.Signed))
+
+
+    STORECLEAN_FUNC = lltype.Ptr(lltype.FuncType([lltype.Ptr(DICT),
+                                                  lltype.Signed,
+                                                  lltype.Signed],
+                                                 lltype.Void))
+
+    for name, T in [('byte', rffi.UCHAR),
+                    ('short', rffi.USHORT),
+                    ('int', rffi.UINT),
+                    ('long', lltype.Unsigned)]:
+        if name == 'int' and not IS_64BIT:
+            continue
+        lookupfn, storecleanfn = new_lookup_functions(LOOKUP_FUNC,
+                                                      STORECLEAN_FUNC, T=T,
+                                                      rtyper=rtyper)
+        setattr(family, '%s_lookup_function' % name, lookupfn)
+        setattr(family, '%s_insert_clean_function' % name, storecleanfn)
+
+def llhelper_or_compile(rtyper, FUNCPTR, ll_func):
+    # the check is for pseudo rtyper from tests
+    if rtyper is None or not hasattr(rtyper, 'annotate_helper_fn'):
+        return llhelper(FUNCPTR, ll_func)
+    else:
+        return rtyper.annotate_helper_fn(ll_func, FUNCPTR.TO.ARGS)
+
+class LookupFamily:
+    def _freeze_(self):
+        return True
+
+
+class OrderedDictRepr(AbstractDictRepr):
+
+    def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
+                 custom_eq_hash=None, force_non_null=False):
+        assert not force_non_null
+        self.rtyper = rtyper
+        self.finalized = False
+        self.DICT = lltype.GcForwardReference()
+        self.lowleveltype = lltype.Ptr(self.DICT)
+        self.custom_eq_hash = custom_eq_hash is not None
+        if not isinstance(key_repr, rmodel.Repr):  # not computed yet, done by setup()
+            assert callable(key_repr)
+            self._key_repr_computer = key_repr
+        else:
+            self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr)
+        if not isinstance(value_repr, rmodel.Repr):  # not computed yet, done by setup()
+            assert callable(value_repr)
+            self._value_repr_computer = value_repr
+        else:
+            self.external_value_repr, self.value_repr = self.pickrepr(value_repr)
+        self.dictkey = dictkey
+        self.dictvalue = dictvalue
+        self.dict_cache = {}
+        self._custom_eq_hash_repr = custom_eq_hash
+        # setup() needs to be called to finish this initialization
+
+    def _externalvsinternal(self, rtyper, item_repr):
+        return rmodel.externalvsinternal(self.rtyper, item_repr)
+
+    def _setup_repr(self):
+        if 'key_repr' not in self.__dict__:
+            key_repr = self._key_repr_computer()
+            self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr)
+        if 'value_repr' not in self.__dict__:
+            self.external_value_repr, self.value_repr = self.pickrepr(self._value_repr_computer())
+        if isinstance(self.DICT, lltype.GcForwardReference):
+            DICTKEY = self.key_repr.lowleveltype
+            DICTVALUE = self.value_repr.lowleveltype
+            # * we need an explicit flag if the key and the value is not
+            #   able to store dummy values
+            s_key   = self.dictkey.s_value
+            s_value = self.dictvalue.s_value
+            kwd = {}
+            if self.custom_eq_hash:
+                self.r_rdict_eqfn, self.r_rdict_hashfn = (
+                    self._custom_eq_hash_repr())
+                kwd['get_custom_eq_hash'] = self._custom_eq_hash_repr
+            else:
+                kwd['ll_hash_function'] = self.key_repr.get_ll_hash_function()
+                kwd['ll_eq_function'] = self.key_repr.get_ll_eq_function()
+                kwd['ll_fasthash_function'] = self.key_repr.get_ll_fasthash_function()
+            kwd['dummykeyobj'] = self.key_repr.get_ll_dummyval_obj(self.rtyper,
+                                                                   s_key)
+            kwd['dummyvalueobj'] = self.value_repr.get_ll_dummyval_obj(
+                self.rtyper, s_value)
+
+            kwd['setup_lookup_funcs'] = False
+            get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT,
+                        rtyper=self.rtyper, **kwd)
+
+    def _setup_repr_final(self):
+        if not self.finalized:
+            family = self.lowleveltype.TO.lookup_family
+            _setup_lookup_funcs(self.lowleveltype.TO, self.rtyper, family)
+            self.finalized = True
+
+
+    def convert_const(self, dictobj):
+        from rpython.rtyper.lltypesystem import llmemory
+        # get object from bound dict methods
+        #dictobj = getattr(dictobj, '__self__', dictobj)
+        if dictobj is None:
+            return lltype.nullptr(self.DICT)
+        if not isinstance(dictobj, (dict, objectmodel.r_dict)):
+            raise TypeError("expected a dict: %r" % (dictobj,))
+        try:
+            key = Constant(dictobj)
+            return self.dict_cache[key]
+        except KeyError:
+            self.setup()
+            self.setup_final()
+            l_dict = ll_newdict_size(self.DICT, len(dictobj))
+            self.dict_cache[key] = l_dict
+            r_key = self.key_repr
+            if r_key.lowleveltype == llmemory.Address:
+                raise TypeError("No prebuilt dicts of address keys")
+            r_value = self.value_repr
+            if isinstance(dictobj, objectmodel.r_dict):
+                if self.r_rdict_eqfn.lowleveltype != lltype.Void:
+                    l_fn = self.r_rdict_eqfn.convert_const(dictobj.key_eq)
+                    l_dict.fnkeyeq = l_fn
+                if self.r_rdict_hashfn.lowleveltype != lltype.Void:
+                    l_fn = self.r_rdict_hashfn.convert_const(dictobj.key_hash)
+                    l_dict.fnkeyhash = l_fn
+
+                for dictkeycontainer, dictvalue in dictobj._dict.items():
+                    llkey = r_key.convert_const(dictkeycontainer.key)
+                    llvalue = r_value.convert_const(dictvalue)
+                    _ll_dict_insertclean(l_dict, llkey, llvalue,
+                                         dictkeycontainer.hash)
+                return l_dict
+
+            else:
+                for dictkey, dictvalue in dictobj.items():
+                    llkey = r_key.convert_const(dictkey)
+                    llvalue = r_value.convert_const(dictvalue)
+                    _ll_dict_insertclean(l_dict, llkey, llvalue,
+                                         l_dict.keyhash(llkey))
+                return l_dict
+
+    def rtype_len(self, hop):
+        v_dict, = hop.inputargs(self)
+        return hop.gendirectcall(ll_dict_len, v_dict)
+
+    def rtype_bool(self, hop):
+        v_dict, = hop.inputargs(self)
+        return hop.gendirectcall(ll_dict_bool, v_dict)
+
+    def make_iterator_repr(self, *variant):
+        return DictIteratorRepr(self, *variant)
+
+    def rtype_method_get(self, hop):
+        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                 self.value_repr)
+        hop.exception_cannot_occur()
+        v_res = hop.gendirectcall(ll_dict_get, v_dict, v_key, v_default)
+        return self.recast_value(hop.llops, v_res)
+
+    def rtype_method_setdefault(self, hop):
+        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                 self.value_repr)
+        hop.exception_cannot_occur()
+        v_res = hop.gendirectcall(ll_dict_setdefault, v_dict, v_key, v_default)
+        return self.recast_value(hop.llops, v_res)
+
+    def rtype_method_copy(self, hop):
+        v_dict, = hop.inputargs(self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_copy, v_dict)
+
+    def rtype_method_update(self, hop):
+        v_dic1, v_dic2 = hop.inputargs(self, self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_update, v_dic1, v_dic2)
+
+    def _rtype_method_kvi(self, hop, ll_func):
+        v_dic, = hop.inputargs(self)
+        r_list = hop.r_result
+        cLIST = hop.inputconst(lltype.Void, r_list.lowleveltype.TO)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_func, cLIST, v_dic)
+
+    def rtype_method_keys(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_keys)
+
+    def rtype_method_values(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_values)
+
+    def rtype_method_items(self, hop):
+        return self._rtype_method_kvi(hop, ll_dict_items)
+
+    def rtype_method_iterkeys(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "keys").newiter(hop)
+
+    def rtype_method_itervalues(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "values").newiter(hop)
+
+    def rtype_method_iteritems(self, hop):
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(self, "items").newiter(hop)
+
+    def rtype_method_clear(self, hop):
+        v_dict, = hop.inputargs(self)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_dict_clear, v_dict)
+
+    def rtype_method_popitem(self, hop):
+        v_dict, = hop.inputargs(self)
+        r_tuple = hop.r_result
+        cTUPLE = hop.inputconst(lltype.Void, r_tuple.lowleveltype)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_popitem, cTUPLE, v_dict)
+
+    def rtype_method_pop(self, hop):
+        if hop.nb_args == 2:
+            v_args = hop.inputargs(self, self.key_repr)
+            target = ll_dict_pop
+        elif hop.nb_args == 3:
+            v_args = hop.inputargs(self, self.key_repr, self.value_repr)
+            target = ll_dict_pop_default
+        hop.exception_is_here()
+        v_res = hop.gendirectcall(target, *v_args)
+        return self.recast_value(hop.llops, v_res)
+
+class __extend__(pairtype(OrderedDictRepr, rmodel.Repr)):
+
+    def rtype_getitem((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        if not r_dict.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)   # record that we know about it
+        hop.exception_is_here()
+        v_res = hop.gendirectcall(ll_dict_getitem, v_dict, v_key)
+        return r_dict.recast_value(hop.llops, v_res)
+
+    def rtype_delitem((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        if not r_dict.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)   # record that we know about it
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_delitem, v_dict, v_key)
+
+    def rtype_setitem((r_dict, r_key), hop):
+        v_dict, v_key, v_value = hop.inputargs(r_dict, r_dict.key_repr, r_dict.value_repr)
+        if r_dict.custom_eq_hash:
+            hop.exception_is_here()
+        else:
+            hop.exception_cannot_occur()
+        hop.gendirectcall(ll_dict_setitem, v_dict, v_key, v_value)
+
+    def rtype_contains((r_dict, r_key), hop):
+        v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_dict_contains, v_dict, v_key)
+
+class __extend__(pairtype(OrderedDictRepr, OrderedDictRepr)):
+    def convert_from_to((r_dict1, r_dict2), v, llops):
+        # check that we don't convert from Dicts with
+        # different key/value types
+        if r_dict1.dictkey is None or r_dict2.dictkey is None:
+            return NotImplemented
+        if r_dict1.dictkey is not r_dict2.dictkey:
+            return NotImplemented
+        if r_dict1.dictvalue is None or r_dict2.dictvalue is None:
+            return NotImplemented
+        if r_dict1.dictvalue is not r_dict2.dictvalue:
+            return NotImplemented
+        return v
+
+# ____________________________________________________________
+#
+#  Low-level methods.  These can be run for testing, but are meant to
+#  be direct_call'ed from rtyped flow graphs, which means that they will
+#  get flowed and annotated, mostly with SomePtr.
+
+DICTINDEX_LONG = lltype.Ptr(lltype.GcArray(lltype.Unsigned))
+DICTINDEX_INT = lltype.Ptr(lltype.GcArray(rffi.UINT))
+DICTINDEX_SHORT = lltype.Ptr(lltype.GcArray(rffi.USHORT))
+DICTINDEX_BYTE = lltype.Ptr(lltype.GcArray(rffi.UCHAR))
+
+IS_64BIT = sys.maxint != 2 ** 31 - 1
+
+if IS_64BIT:
+    FUNC_BYTE, FUNC_SHORT, FUNC_INT, FUNC_LONG = range(4)
+else:
+    FUNC_BYTE, FUNC_SHORT, FUNC_LONG = range(3)
+
+def ll_malloc_indexes_and_choose_lookup(d, n):
+    if n <= 256:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_BYTE.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_BYTE
+    elif n <= 65536:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_SHORT.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_SHORT
+    elif IS_64BIT and n <= 2 ** 32:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_INT.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_INT
+    else:
+        d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
+                                           lltype.malloc(DICTINDEX_LONG.TO, n,
+                                                         zero=True))
+        d.lookup_function_no = FUNC_LONG
+
+def ll_call_insert_clean_function(d, hash, i):
+    DICT = lltype.typeOf(d).TO
+    if d.lookup_function_no == FUNC_BYTE:
+        DICT.lookup_family.byte_insert_clean_function(d, hash, i)
+    elif d.lookup_function_no == FUNC_SHORT:
+        DICT.lookup_family.short_insert_clean_function(d, hash, i)
+    elif IS_64BIT and d.lookup_function_no == FUNC_INT:
+        DICT.lookup_family.int_insert_clean_function(d, hash, i)
+    elif d.lookup_function_no == FUNC_LONG:
+        DICT.lookup_family.long_insert_clean_function(d, hash, i)
+    else:
+        assert False
+
+def ll_valid_from_flag(entries, i):
+    return entries[i].f_valid
+
+def ll_valid_from_key(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    return entries[i].key != dummy
+
+def ll_valid_from_value(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    return entries[i].value != dummy
+
+def ll_mark_deleted_in_flag(entries, i):
+    entries[i].f_valid = False
+
+def ll_mark_deleted_in_key(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    entries[i].key = dummy
+
+def ll_mark_deleted_in_value(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    dummy = ENTRIES.dummy_obj.ll_dummy_value
+    entries[i].value = dummy
+
+def ll_hash_from_cache(entries, i):
+    return entries[i].f_hash
+
+def ll_hash_recomputed(entries, i):
+    ENTRIES = lltype.typeOf(entries).TO
+    return ENTRIES.fasthashfn(entries[i].key)
+
+def ll_keyhash_custom(d, key):
+    DICT = lltype.typeOf(d).TO
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+
+def ll_keyeq_custom(d, key1, key2):
+    DICT = lltype.typeOf(d).TO
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+
+def ll_dict_len(d):
+    return d.num_items
+
+def ll_dict_bool(d):
+    # check if a dict is True, allowing for None
+    return bool(d) and d.num_items != 0
+
+def ll_dict_getitem(d, key):
+    index = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
+    if index != -1:
+        return d.entries[index].value
+    else:
+        raise KeyError
+
+def ll_dict_setitem(d, key, value):
+    hash = d.keyhash(key)
+    index = d.lookup_function(d, key, hash, FLAG_STORE)
+    return _ll_dict_setitem_lookup_done(d, key, value, hash, index)
+
+# It may be safe to look inside always, it has a few branches though, and their
+# frequencies needs to be investigated.
+ at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
+def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    if i >= 0:
+        entry = d.entries[i]
+        entry.value = value
+    else:
+        if len(d.entries) == d.num_used_items:
+            if ll_dict_grow(d):
+                ll_call_insert_clean_function(d, hash, d.num_used_items)
+        entry = d.entries[d.num_used_items]
+        entry.key = key
+        entry.value = value
+        if hasattr(ENTRY, 'f_hash'):
+            entry.f_hash = hash
+        if hasattr(ENTRY, 'f_valid'):
+            entry.f_valid = True
+        d.num_used_items += 1
+        d.num_items += 1
+        rc = d.resize_counter - 3
+        if rc <= 0:
+            ll_dict_resize(d)
+            rc = d.resize_counter - 3
+            ll_assert(rc > 0, "ll_dict_resize failed?")
+        d.resize_counter = rc
+
+def _ll_dict_insertclean(d, key, value, hash):
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    ll_call_insert_clean_function(d, hash, d.num_used_items)
+    entry = d.entries[d.num_used_items]
+    entry.key = key
+    entry.value = value
+    if hasattr(ENTRY, 'f_hash'):
+        entry.f_hash = hash
+    if hasattr(ENTRY, 'f_valid'):
+        entry.f_valid = True
+    d.num_used_items += 1
+    d.num_items += 1
+    rc = d.resize_counter - 3
+    d.resize_counter = rc
+
+def _ll_len_of_d_indexes(d):
+    # xxx Haaaack: returns len(d.indexes).  Works independently of
+    # the exact type pointed to by d, using a forced cast...
+    return len(rffi.cast(DICTINDEX_BYTE, d.indexes))
+
+def _overallocate_entries_len(baselen):
+    # This over-allocates proportional to the list size, making room
+    # for additional growth.  The over-allocation is mild, but is
+    # enough to give linear-time amortized behavior over a long
+    # sequence of appends() in the presence of a poorly-performing
+    # system malloc().
+    # The growth pattern is:  0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
+    newsize = baselen + 1
+    if newsize < 9:
+        some = 3
+    else:
+        some = 6
+    some += newsize >> 3
+    return newsize + some
+
+ at jit.dont_look_inside
+def ll_dict_grow(d):
+    if d.num_items < d.num_used_items // 4:
+        ll_dict_remove_deleted_items(d)
+        return True
+
+    new_allocated = _overallocate_entries_len(len(d.entries))
+
+    # Detect an obscure case where the indexes numeric type is too
+    # small to store all the entry indexes
+    if (max(128, _ll_len_of_d_indexes(d)) - new_allocated
+                   < MIN_INDEXES_MINUS_ENTRIES):
+        ll_dict_remove_deleted_items(d)
+        return True
+
+    newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+    rgc.ll_arraycopy(d.entries, newitems, 0, 0, len(d.entries))
+    d.entries = newitems
+    return False
+
+def ll_dict_remove_deleted_items(d):
+    new_allocated = _overallocate_entries_len(d.num_items)
+    if new_allocated < len(d.entries) // 2:
+        newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+    else:
+        newitems = d.entries
+    #
+    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
+    isrc = 0
+    idst = 0
+    while isrc < len(d.entries):
+        if d.entries.valid(isrc):
+            src = d.entries[isrc]
+            dst = newitems[idst]
+            dst.key = src.key
+            dst.value = src.value
+            if hasattr(ENTRY, 'f_hash'):
+                dst.f_hash = src.f_hash
+            if hasattr(ENTRY, 'f_valid'):
+                assert src.f_valid
+                dst.f_valid = True
+            idst += 1
+        isrc += 1
+    d.entries = newitems
+    assert d.num_items == idst
+    d.num_used_items = idst
+
+    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+
+
+def ll_dict_delitem(d, key):
+    index = d.lookup_function(d, key, d.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        raise KeyError
+    _ll_dict_del(d, index)
+
+ at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
+def _ll_dict_del(d, index):
+    d.entries.mark_deleted(index)
+    d.num_items -= 1
+    # clear the key and the value if they are GC pointers
+    ENTRIES = lltype.typeOf(d.entries).TO
+    ENTRY = ENTRIES.OF
+    entry = d.entries[index]
+    if ENTRIES.must_clear_key:
+        entry.key = lltype.nullptr(ENTRY.key.TO)
+    if ENTRIES.must_clear_value:
+        entry.value = lltype.nullptr(ENTRY.value.TO)
+    #
+    # The rest is commented out: like CPython we no longer shrink the
+    # dictionary here.  It may shrink later if we try to append a number
+    # of new items to it.  Unsure if this behavior was designed in
+    # CPython or is accidental.  A design reason would be that if you
+    # delete all items in a dictionary (e.g. with a series of
+    # popitem()), then CPython avoids shrinking the table several times.
+    #num_entries = len(d.entries)
+    #if num_entries > DICT_INITSIZE and d.num_items <= num_entries / 4:
+    #    ll_dict_resize(d)
+    # A previous xxx: move the size checking and resize into a single
+    # call which is opaque to the JIT when the dict isn't virtual, to
+    # avoid extra branches.
+
+def ll_dict_resize(d):
+    # make a 'new_size' estimate and shrink it if there are many
+    # deleted entry markers.  See CPython for why it is a good idea to
+    # quadruple the dictionary size as long as it's not too big.
+    num_items = d.num_items
+    if num_items > 50000:
+        new_estimate = num_items * 2
+    else:
+        new_estimate = num_items * 4
+    new_size = DICT_INITSIZE
+    while new_size <= new_estimate:
+        new_size *= 2
+
+    if new_size < _ll_len_of_d_indexes(d):
+        ll_dict_remove_deleted_items(d)
+    else:
+        ll_dict_reindex(d, new_size)
+ll_dict_resize.oopspec = 'dict.resize(d)'
+
+def ll_dict_reindex(d, new_size):
+    ll_malloc_indexes_and_choose_lookup(d, new_size)
+    d.resize_counter = new_size * 2 - d.num_items * 3
+    assert d.resize_counter > 0
+    #
+    entries = d.entries
+    i = 0
+    while i < d.num_used_items:
+        if entries.valid(i):
+            hash = entries.hash(i)
+            ll_call_insert_clean_function(d, hash, i)
+        i += 1
+    #old_entries.delete() XXXX!
+
+# ------- a port of CPython's dictobject.c's lookdict implementation -------
+PERTURB_SHIFT = 5
+
+FREE = 0
+DELETED = 1
+VALID_OFFSET = 2
+MIN_INDEXES_MINUS_ENTRIES = VALID_OFFSET + 1
+
+FLAG_LOOKUP = 0
+FLAG_STORE = 1
+FLAG_DELETE = 2
+FLAG_DELETE_TRY_HARD = 3
+
+def new_lookup_functions(LOOKUP_FUNC, STORECLEAN_FUNC, T, rtyper=None):
+    INDEXES = lltype.Ptr(lltype.GcArray(T))
+
+    def ll_kill_something(d):
+        i = 0
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        while True:
+            index = rffi.cast(lltype.Signed, indexes[i])
+            if index >= VALID_OFFSET:
+                indexes[i] = rffi.cast(T, DELETED)
+                return index
+            i += 1
+
+    @jit.look_inside_iff(lambda d, key, hash, store_flag:
+                         jit.isvirtual(d) and jit.isconstant(key))
+    def ll_dict_lookup(d, key, hash, store_flag):
+        entries = d.entries
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        mask = len(indexes) - 1
+        i = r_uint(hash & mask)
+        # do the first try before any looping
+        ENTRIES = lltype.typeOf(entries).TO
+        direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
+        index = rffi.cast(lltype.Signed, indexes[intmask(i)])
+        if index >= VALID_OFFSET:
+            checkingkey = entries[index - VALID_OFFSET].key
+            if direct_compare and checkingkey == key:
+                if store_flag == FLAG_DELETE:
+                    indexes[i] = rffi.cast(T, DELETED)
+                return index - VALID_OFFSET   # found the entry
+            if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
+                # correct hash, maybe the key is e.g. a different pointer to
+                # an equal object
+                found = d.keyeq(checkingkey, key)
+                #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found)
+                if d.paranoia:
+                    if (entries != d.entries or lltype.cast_opaque_ptr(llmemory.GCREF, indexes) != d.indexes or
+                        not entries.valid(index - VALID_OFFSET) or
+                        entries[index - VALID_OFFSET].key != checkingkey):
+                        # the compare did major nasty stuff to the dict: start over
+                        return ll_dict_lookup(d, key, hash, store_flag)
+                if found:
+                    if store_flag == FLAG_DELETE:
+                        indexes[i] = rffi.cast(T, DELETED)
+                    return index - VALID_OFFSET
+            deletedslot = -1
+        elif index == DELETED:
+            deletedslot = intmask(i)
+        else:
+            # pristine entry -- lookup failed
+            if store_flag == FLAG_STORE:
+                indexes[i] = rffi.cast(T, d.num_used_items + VALID_OFFSET)
+            elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
+                return ll_kill_something(d)
+            return -1
+
+        # In the loop, a deleted entry (everused and not valid) is by far
+        # (factor of 100s) the least likely outcome, so test for that last.
+        perturb = r_uint(hash)
+        while 1:
+            # compute the next index using unsigned arithmetic
+            i = (i << 2) + i + perturb + 1
+            i = i & mask
+            index = rffi.cast(lltype.Signed, indexes[intmask(i)])
+            if index == FREE:
+                if store_flag == FLAG_STORE:
+                    if deletedslot == -1:
+                        deletedslot = intmask(i)
+                    indexes[deletedslot] = rffi.cast(T, d.num_used_items +
+                                                     VALID_OFFSET)
+                elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
+                    return ll_kill_something(d)
+                return -1
+            elif index >= VALID_OFFSET:
+                checkingkey = entries[index - VALID_OFFSET].key
+                if direct_compare and checkingkey == key:
+                    if store_flag == FLAG_DELETE:
+                        indexes[i] = rffi.cast(T, DELETED)
+                    return index - VALID_OFFSET   # found the entry
+                if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
+                    # correct hash, maybe the key is e.g. a different pointer to
+                    # an equal object
+                    found = d.keyeq(checkingkey, key)
+                    if d.paranoia:
+                        if (entries != d.entries or lltype.cast_opaque_ptr(llmemory.GCREF, indexes) != d.indexes or
+                            not entries.valid(index - VALID_OFFSET) or
+                            entries[index - VALID_OFFSET].key != checkingkey):
+                            # the compare did major nasty stuff to the dict: start over
+                            return ll_dict_lookup(d, key, hash, store_flag)
+                    if found:
+                        if store_flag == FLAG_DELETE:
+                            indexes[i] = rffi.cast(T, DELETED)
+                        return index - VALID_OFFSET
+            elif deletedslot == -1:
+                deletedslot = intmask(i)
+            perturb >>= PERTURB_SHIFT
+
+    def ll_dict_store_clean(d, hash, index):
+        # a simplified version of ll_dict_lookup() which assumes that the
+        # key is new, and the dictionary doesn't contain deleted entries.
+        # It only finds the next free slot for the given hash.
+        indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+        mask = len(indexes) - 1
+        i = r_uint(hash & mask)
+        perturb = r_uint(hash)
+        while rffi.cast(lltype.Signed, indexes[i]) != 0:
+            i = (i << 2) + i + perturb + 1
+            i = i & mask
+            perturb >>= PERTURB_SHIFT
+        indexes[i] = rffi.cast(T, index + VALID_OFFSET)
+
+    return (llhelper_or_compile(rtyper, LOOKUP_FUNC, ll_dict_lookup),
+            llhelper_or_compile(rtyper, STORECLEAN_FUNC, ll_dict_store_clean))
+
+# ____________________________________________________________
+#
+#  Irregular operations.
+
+DICT_INITSIZE = 8
+
+def ll_newdict(DICT):
+    d = DICT.allocate()
+    d.entries = DICT.lookup_family.empty_array
+    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = DICT_INITSIZE * 2
+    return d
+
+def ll_newdict_size(DICT, orig_length_estimate):
+    length_estimate = (orig_length_estimate // 2) * 3
+    n = DICT_INITSIZE
+    while n < length_estimate:
+        n *= 2
+    d = DICT.allocate()
+    d.entries = DICT.entries.TO.allocate(orig_length_estimate)
+    ll_malloc_indexes_and_choose_lookup(d, n)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = n * 2
+    return d
+
+# rpython.memory.lldict uses a dict based on Struct and Array
+# instead of GcStruct and GcArray, which is done by using different
+# 'allocate' and 'delete' adtmethod implementations than the ones below
+def _ll_malloc_dict(DICT):
+    return lltype.malloc(DICT)
+def _ll_malloc_entries(ENTRIES, n):
+    return lltype.malloc(ENTRIES, n, zero=True)
+def _ll_free_entries(entries):
+    pass
+
+
+def rtype_r_dict(hop):
+    r_dict = hop.r_result
+    if not r_dict.custom_eq_hash:
+        raise TyperError("r_dict() call does not return an r_dict instance")
+    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
+    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
+    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
+    hop.exception_cannot_occur()
+    v_result = hop.gendirectcall(ll_newdict, cDICT)
+    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
+        hop.genop('setfield', [v_result, cname, v_eqfn])
+    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
+        hop.genop('setfield', [v_result, cname, v_hashfn])
+    return v_result
+
+# ____________________________________________________________
+#
+#  Iteration.
+
+def get_ll_dictiter(DICTPTR):
+    return lltype.Ptr(lltype.GcStruct('dictiter',
+                                      ('dict', DICTPTR),
+                                      ('index', lltype.Signed)))
+
+class DictIteratorRepr(AbstractDictIteratorRepr):
+
+    def __init__(self, r_dict, variant="keys"):
+        self.r_dict = r_dict
+        self.variant = variant
+        self.lowleveltype = get_ll_dictiter(r_dict.lowleveltype)
+        self.ll_dictiter = ll_dictiter
+        self.ll_dictnext = ll_dictnext_group[variant]
+
+
+def ll_dictiter(ITERPTR, d):
+    iter = lltype.malloc(ITERPTR.TO)
+    iter.dict = d
+    iter.index = 0
+    return iter
+
+def _make_ll_dictnext(kind):
+    # make three versions of the following function: keys, values, items
+    @jit.look_inside_iff(lambda RETURNTYPE, iter: jit.isvirtual(iter)
+                         and (iter.dict is None or
+                              jit.isvirtual(iter.dict)))
+    @jit.oopspec("dictiter.next%s(iter)" % kind)
+    def ll_dictnext(RETURNTYPE, iter):
+        # note that RETURNTYPE is None for keys and values
+        dict = iter.dict
+        if not dict:
+            raise StopIteration
+
+        entries = dict.entries
+        index = iter.index
+        assert index >= 0
+        entries_len = dict.num_used_items
+        while index < entries_len:
+            entry = entries[index]
+            is_valid = entries.valid(index)
+            index = index + 1
+            if is_valid:
+                iter.index = index
+                if RETURNTYPE is lltype.Void:
+                    return None
+                elif kind == 'items':
+                    r = lltype.malloc(RETURNTYPE.TO)
+                    r.item0 = recast(RETURNTYPE.TO.item0, entry.key)
+                    r.item1 = recast(RETURNTYPE.TO.item1, entry.value)
+                    return r
+                elif kind == 'keys':
+                    return entry.key
+                elif kind == 'values':
+                    return entry.value
+
+        # clear the reference to the dict and prevent restarts
+        iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
+        raise StopIteration
+
+    return ll_dictnext
+
+ll_dictnext_group = {'keys'  : _make_ll_dictnext('keys'),
+                     'values': _make_ll_dictnext('values'),
+                     'items' : _make_ll_dictnext('items')}
+
+# _____________________________________________________________
+# methods
+
+def ll_dict_get(dict, key, default):
+    index = dict.lookup_function(dict, key, dict.keyhash(key), FLAG_LOOKUP)
+    if index == -1:
+        return default
+    else:
+        return dict.entries[index].value
+
+def ll_dict_setdefault(dict, key, default):
+    hash = dict.keyhash(key)
+    index = dict.lookup_function(dict, key, hash, FLAG_STORE)
+    if index == -1:
+        _ll_dict_setitem_lookup_done(dict, key, default, hash, -1)
+        return default
+    else:
+        return dict.entries[index].value
+
+def ll_dict_copy(dict):
+    DICT = lltype.typeOf(dict).TO
+    newdict = DICT.allocate()
+    newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
+
+    newdict.num_items = dict.num_items
+    newdict.num_used_items = dict.num_used_items
+    if hasattr(DICT, 'fnkeyeq'):
+        newdict.fnkeyeq = dict.fnkeyeq
+    if hasattr(DICT, 'fnkeyhash'):
+        newdict.fnkeyhash = dict.fnkeyhash
+
+    i = 0
+    while i < newdict.num_used_items:
+        d_entry = newdict.entries[i]
+        entry = dict.entries[i]
+        ENTRY = lltype.typeOf(newdict.entries).TO.OF
+        d_entry.key = entry.key
+        if hasattr(ENTRY, 'f_valid'):
+            d_entry.f_valid = entry.f_valid
+        d_entry.value = entry.value
+        if hasattr(ENTRY, 'f_hash'):
+            d_entry.f_hash = entry.f_hash
+        i += 1
+
+    ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
+    return newdict
+ll_dict_copy.oopspec = 'dict.copy(dict)'
+
+def ll_dict_clear(d):
+    if d.num_used_items == 0:
+        return
+    DICT = lltype.typeOf(d).TO
+    old_entries = d.entries
+    d.entries = DICT.lookup_family.empty_array
+    ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
+    d.num_items = 0
+    d.num_used_items = 0
+    d.resize_counter = DICT_INITSIZE * 2
+    # old_entries.delete() XXX
+ll_dict_clear.oopspec = 'dict.clear(d)'
+
+def ll_dict_update(dic1, dic2):
+    i = 0
+    while i < dic2.num_used_items:
+        entries = dic2.entries
+        if entries.valid(i):
+            entry = entries[i]
+            hash = entries.hash(i)
+            key = entry.key
+            value = entry.value
+            index = dic1.lookup_function(dic1, key, hash, FLAG_STORE)
+            _ll_dict_setitem_lookup_done(dic1, key, value, hash, index)
+        i += 1
+ll_dict_update.oopspec = 'dict.update(dic1, dic2)'
+
+# this is an implementation of keys(), values() and items()
+# in a single function.
+# note that by specialization on func, three different
+# and very efficient functions are created.
+
+def recast(P, v):
+    if isinstance(P, lltype.Ptr):
+        return lltype.cast_pointer(P, v)
+    else:
+        return v
+
+def _make_ll_keys_values_items(kind):
+    def ll_kvi(LIST, dic):
+        res = LIST.ll_newlist(dic.num_items)
+        entries = dic.entries
+        dlen = dic.num_used_items
+        items = res.ll_items()
+        i = 0
+        p = 0
+        while i < dlen:
+            if entries.valid(i):
+                ELEM = lltype.typeOf(items).TO.OF
+                if ELEM is not lltype.Void:
+                    entry = entries[i]
+                    if kind == 'items':
+                        r = lltype.malloc(ELEM.TO)
+                        r.item0 = recast(ELEM.TO.item0, entry.key)
+                        r.item1 = recast(ELEM.TO.item1, entry.value)
+                        items[p] = r
+                    elif kind == 'keys':
+                        items[p] = recast(ELEM, entry.key)
+                    elif kind == 'values':
+                        items[p] = recast(ELEM, entry.value)
+                p += 1
+            i += 1
+        assert p == res.ll_length()
+        return res
+    ll_kvi.oopspec = 'dict.%s(dic)' % kind
+    return ll_kvi
+
+ll_dict_keys   = _make_ll_keys_values_items('keys')
+ll_dict_values = _make_ll_keys_values_items('values')
+ll_dict_items  = _make_ll_keys_values_items('items')
+
+def ll_dict_contains(d, key):
+    i = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
+    return i != -1
+
+def _ll_getnextitem(dic):
+    if dic.num_items == 0:
+        raise KeyError
+
+    entries = dic.entries
+
+    while True:
+        i = dic.num_used_items - 1
+        if entries.valid(i):
+            break
+        dic.num_used_items -= 1
+
+    key = entries[i].key
+    index = dic.lookup_function(dic, key, entries.hash(i),
+                                FLAG_DELETE_TRY_HARD)
+    # if the lookup function returned me a random strange thing,
+    # don't care about deleting the item
+    if index == dic.num_used_items - 1:
+        dic.num_used_items -= 1
+    else:
+        assert index != -1
+    return index
+
+def ll_dict_popitem(ELEM, dic):
+    i = _ll_getnextitem(dic)
+    entry = dic.entries[i]
+    r = lltype.malloc(ELEM.TO)
+    r.item0 = recast(ELEM.TO.item0, entry.key)
+    r.item1 = recast(ELEM.TO.item1, entry.value)
+    _ll_dict_del(dic, i)
+    return r
+
+def ll_dict_pop(dic, key):
+    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        raise KeyError
+    value = dic.entries[index].value
+    _ll_dict_del(dic, index)
+    return value
+
+def ll_dict_pop_default(dic, key, dfl):
+    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    if index == -1:
+        return dfl
+    value = dic.entries[index].value
+    _ll_dict_del(dic, index)
+    return value
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -1,3 +1,5 @@
+from collections import OrderedDict
+
 from rpython.annotator import model as annmodel
 from rpython.flowspace.model import Constant
 from rpython.rlib import rarithmetic, objectmodel
@@ -726,10 +728,29 @@
 
     raise TyperError("hasattr is only suported on a constant")
 
+def rtype_ordered_dict(hop):
+    from rpython.rtyper.lltypesystem.rordereddict import ll_newdict
+
+    hop.exception_cannot_occur()
+    r_dict = hop.r_result
+    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
+    v_result = hop.gendirectcall(ll_newdict, cDICT)
+    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
+    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
+    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
+        hop.genop('setfield', [v_result, cname, v_eqfn])
+    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
+        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
+        hop.genop('setfield', [v_result, cname, v_hashfn])
+    return v_result
+
 BUILTIN_TYPER[objectmodel.instantiate] = rtype_instantiate
 BUILTIN_TYPER[isinstance] = rtype_builtin_isinstance
 BUILTIN_TYPER[hasattr] = rtype_builtin_hasattr
 BUILTIN_TYPER[objectmodel.r_dict] = rtype_r_dict
+BUILTIN_TYPER[OrderedDict] = rtype_ordered_dict
+BUILTIN_TYPER[objectmodel.r_ordereddict] = rtype_ordered_dict
 
 # _________________________________________________________________
 # weakrefs
diff --git a/rpython/rtyper/rdict.py b/rpython/rtyper/rdict.py
--- a/rpython/rtyper/rdict.py
+++ b/rpython/rtyper/rdict.py
@@ -4,8 +4,12 @@
 
 
 class __extend__(annmodel.SomeDict):
+    def get_dict_repr(self):
+        from rpython.rtyper.lltypesystem.rdict import DictRepr
+
+        return DictRepr
+
     def rtyper_makerepr(self, rtyper):
-        from rpython.rtyper.lltypesystem.rdict import DictRepr
         dictkey = self.dictdef.dictkey
         dictvalue = self.dictdef.dictvalue
         s_key = dictkey.s_value
@@ -16,7 +20,7 @@
                                       rtyper.getrepr(dictkey.s_rdict_hashfn))
         else:
             custom_eq_hash = None
-        return DictRepr(rtyper, lambda: rtyper.getrepr(s_key),
+        return self.get_dict_repr()(rtyper, lambda: rtyper.getrepr(s_key),
                         lambda: rtyper.getrepr(s_value), dictkey, dictvalue,
                         custom_eq_hash, force_non_null)
 
@@ -25,6 +29,11 @@
         self.dictdef.dictvalue.dont_change_any_more = True
         return (self.__class__, self.dictdef.dictkey, self.dictdef.dictvalue)
 
+class __extend__(annmodel.SomeOrderedDict):
+    def get_dict_repr(self):
+        from rpython.rtyper.lltypesystem.rordereddict import OrderedDictRepr
+
+        return OrderedDictRepr
 
 class AbstractDictRepr(rmodel.Repr):
 
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -22,11 +22,11 @@
         yield x
 
 
-class TestRdict(BaseRtypingTest):
-
+class BaseTestRDict(BaseRtypingTest):
     def test_dict_creation(self):
         def createdict(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             return d['hello']
 
         res = self.interpret(createdict, [42])
@@ -34,7 +34,8 @@
 
     def test_dict_getitem_setitem(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             d['world'] = i + 1
             return d['hello'] * d['world']
         res = self.interpret(func, [6])
@@ -42,7 +43,8 @@
 
     def test_dict_getitem_keyerror(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             try:
                 return d['world']
             except KeyError:
@@ -52,7 +54,8 @@
 
     def test_dict_del_simple(self):
         def func(i):
-            d = {'hello' : i}
+            d = self.newdict()
+            d['hello'] = i
             d['world'] = i + 1
             del d['hello']
             return len(d)
@@ -61,7 +64,8 @@
 
     def test_dict_clear(self):
         def func(i):
-            d = {'abc': i}
+            d = self.newdict()
+            d['abc'] = i
             d['def'] = i+1
             d.clear()
             d['ghi'] = i+2
@@ -72,7 +76,8 @@
 
     def test_empty_strings(self):
         def func(i):
-            d = {'' : i}
+            d = self.newdict()
+            d[''] = i
             del d['']
             try:
                 d['']
@@ -84,7 +89,8 @@
         assert res == 1
 
         def func(i):
-            d = {'' : i}
+            d = self.newdict()
+            d[''] = i
             del d['']
             d[''] = i + 1
             return len(d)
@@ -94,9 +100,10 @@
     def test_dict_bool(self):
         def func(i):
             if i:
-                d = {}
+                d = self.newdict()
             else:
-                d = {i: i+1}
+                d = self.newdict()
+                d[i] = i+1
             if d:
                 return i
             else:
@@ -106,17 +113,20 @@
 
     def test_contains(self):
         def func(x, y):
-            d = {x: x+1}
+            d = self.newdict()
+            d[x] = x+1
             return y in d
         assert self.interpret(func, [42, 0]) == False
         assert self.interpret(func, [42, 42]) == True
 
     def test_contains_2(self):
-        d = {'5': None, '7': None}
+        d = self.newdict()
+        d['5'] = None
+        d['7'] = None
         def func(x):
             return chr(x) in d
-        #assert self.interpret(func, [ord('5')]) == True
-        #assert self.interpret(func, [ord('6')]) == False
+        assert self.interpret(func, [ord('5')]) == True
+        assert self.interpret(func, [ord('6')]) == False
 
         def func(n):
             return str(n) in d
@@ -124,7 +134,7 @@
 
     def test_dict_iteration(self):
         def func(i, j):
-            d = {}
+            d = self.newdict()
             d['hello'] = i
             d['world'] = j
             k = 1
@@ -136,7 +146,7 @@
 
     def test_dict_itermethods(self):
         def func():
-            d = {}
+            d = self.newdict()
             d['hello'] = 6
             d['world'] = 7
             k1 = k2 = k3 = 1
@@ -151,19 +161,9 @@
         res = self.interpret(func, [])
         assert res == 42 + 42 + 42
 
-    def test_two_dicts_with_different_value_types(self):
-        def func(i):
-            d1 = {}
-            d1['hello'] = i + 1
-            d2 = {}
-            d2['world'] = d1
-            return d2['world']['hello']
-        res = self.interpret(func, [5])
-        assert res == 6
-
     def test_dict_get(self):
         def func():
-            dic = {}
+            dic = self.newdict()
             x1 = dic.get('hi', 42)
             dic['blah'] = 1 # XXX this triggers type determination
             x2 = dic.get('blah', 2)
@@ -174,7 +174,7 @@
     def test_dict_get_empty(self):
         def func():
             # this time without writing to the dict
-            dic = {}
+            dic = self.newdict()
             x1 = dic.get('hi', 42)
             x2 = dic.get('blah', 2)
             return x1 * 10 + x2
@@ -183,14 +183,14 @@
 
     def test_dict_setdefault(self):
         def f():
-            d = {}
+            d = self.newdict()
             d.setdefault('a', 2)
             return d['a']
         res = self.interpret(f, ())
         assert res == 2
 
         def f():
-            d = {}
+            d = self.newdict()
             d.setdefault('a', 2)
             x = d.setdefault('a', -3)
             return x
@@ -200,7 +200,9 @@
     def test_dict_copy(self):
         def func():
             # XXX this does not work if we use chars, only!
-            dic = {'ab':1, 'b':2}
+            dic = self.newdict()
+            dic['ab'] = 1
+            dic['b'] = 2
             d2 = dic.copy()
             ok = 1
             for key in d2:
@@ -215,8 +217,12 @@
 
     def test_dict_update(self):
         def func():
-            dic = {'ab':1000, 'b':200}
-            d2 = {'b':30, 'cb':4}
+            dic = self.newdict()
+            dic['ab'] = 1000
+            dic['b'] = 200
+            d2 = self.newdict()
+            d2['b'] = 30
+            d2['cb'] = 4
             dic.update(d2)
             ok = len(dic) == 3
             sum = ok
@@ -228,7 +234,9 @@
 
     def test_dict_keys(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             keys = dic.keys()
             return ord(keys[0][1]) + ord(keys[1][1]) - 2*ord('0') + len(keys)
         res = self.interpret(func, ())#, view=True)
@@ -240,8 +248,11 @@
         class A(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {A(): 1, A(): 2}
+            dic0 = self.newdict()
+            dic0[Empty()] = 2
+            dic = self.newdict()
+            dic[A()] = 1
+            dic[A()] = 2
             keys = dic.keys()
             return (isinstance(keys[1], A))*2+(isinstance(keys[0],A))
         res = self.interpret(func, [])
@@ -253,8 +264,11 @@
         class A(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {A(): 1, A(): 2}
+            dic0 = self.newdict()
+            dic0[Empty()] = 2
+            dic = self.newdict()
+            dic[A()] = 1
+            dic[A()] = 2
             a = 0
             for k in dic.iterkeys():
                 a += isinstance(k, A)
@@ -264,7 +278,9 @@
 
     def test_dict_values(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             values = dic.values()
             return values[0] + values[1] + len(values)
         res = self.interpret(func, ())
@@ -274,7 +290,9 @@
         class A:
             pass
         def func():
-            dic = {1: A(), 2: A()}
+            dic = self.newdict()
+            dic[1] = A()
+            dic[2] = A()
             vals = dic.values()
             return (isinstance(vals[1], A))*2+(isinstance(vals[0],A))
         res = self.interpret(func, [])
@@ -284,7 +302,9 @@
         class A:
             pass
         def func():
-            dic = {1: A(), 2: A()}
+            dic = self.newdict()
+            dic[1] = A()
+            dic[2] = A()
             a = 0
             for v in dic.itervalues():
                 a += isinstance(v, A)
@@ -300,8 +320,11 @@
         class B(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {B(): A(), B(): A()}
+            dic0 = self.newdict()
+            dic0[Empty()] = A()
+            dic = self.newdict()
+            dic[B()] = A()
+            dic[B()] = A()
             items = dic.items()
             b = 0
             a = 0
@@ -320,8 +343,11 @@
         class B(Empty):
             pass
         def func():
-            dic0 = {Empty(): 2}
-            dic = {B(): A(), B(): A()}
+            dic0 = self.newdict()
+            dic0[Empty()] = A()
+            dic = self.newdict()
+            dic[B()] = A()
+            dic[B()] = A()
             b = 0
             a = 0
             for k, v in dic.iteritems():
@@ -333,7 +359,9 @@
 
     def test_dict_items(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             items = dic.items()
             res = len(items)
             for key, value in items:
@@ -344,13 +372,17 @@
 
     def test_dict_contains(self):
         def func():
-            dic = {' 4':1000, ' 8':200}
+            dic = self.newdict()
+            dic[' 4'] = 1000
+            dic[' 8'] = 200
             return ' 4' in dic and ' 9' not in dic
         res = self.interpret(func, ())
         assert res is True
 
     def test_dict_contains_with_constant_dict(self):
-        dic = {'4':1000, ' 8':200}
+        dic = self.newdict()
+        dic['4'] = 1000
+        dic['8'] = 200
         def func(i):
             return chr(i) in dic
         res = self.interpret(func, [ord('4')])
@@ -367,7 +399,9 @@
             a = A()
             a.d = None
             if n > 0:
-                a.d = {str(n): 1, "42": 2}
+                a.d = self.newdict()
+                a.d[str(n)] = 1
+                a.d["42"] = 2
                 del a.d["42"]
             return negate(a.d)
         res = self.interpret(func, [10])
@@ -379,7 +413,8 @@
 
     def test_int_dict(self):
         def func(a, b):
-            dic = {12: 34}
+            dic = self.newdict()
+            dic[12] = 34
             dic[a] = 1000
             return dic.get(b, -123)
         res = self.interpret(func, [12, 12])
@@ -403,7 +438,7 @@
         def f():
             a = A()
             b = B()
-            d = {}
+            d = self.newdict()
             d[b] = 7
             d[a] = 3
             return len(d) + d[a] + d[b]
@@ -411,7 +446,9 @@
         assert res == 12
 
     def test_captured_get(self):
-        get = {1:2}.get
+        d = self.newdict()
+        d[1] = 2
+        get = d.get
         def f():
             return get(1, 3)+get(2, 4)
         res = self.interpret(f, [])
@@ -431,40 +468,21 @@
         def f():
             lst = [A()]
             res1 = A() in lst
-            d2 = {B(): None, B(): None}
+            d2 = self.newdict()
+            d2[B()] = None
+            d2[B()] = None
             return res1+len(d2)
         res = self.interpret(f, [])
         assert res == 2
 
-
-    def test_type_erase(self):
-        class A(object):
-            pass
-        class B(object):
-            pass
-
-        def f():
-            return {A(): B()}, {B(): A()}
-
-        t = TranslationContext()
-        s = t.buildannotator().build_types(f, [])
-        rtyper = t.buildrtyper()
-        rtyper.specialize()
-
-        s_AB_dic = s.items[0]
-        s_BA_dic = s.items[1]
-
-        r_AB_dic = rtyper.getrepr(s_AB_dic)
-        r_BA_dic = rtyper.getrepr(s_AB_dic)
-
-        assert r_AB_dic.lowleveltype == r_BA_dic.lowleveltype
-
     def test_identity_hash_is_fast(self):
         class A(object):
             pass
 
         def f():
-            return {A(): 1}
+            d = self.newdict()
+            d[A()] = 1
+            return d
 
         t = TranslationContext()
         s = t.buildannotator().build_types(f, [])
@@ -476,7 +494,7 @@
 
     def test_tuple_dict(self):
         def f(i):
-            d = {}
+            d = self.newdict()
             d[(1, 4.5, (str(i), 2), 2)] = 4
             d[(1, 4.5, (str(i), 2), 3)] = 6
             return d[(1, 4.5, (str(i), 2), i)]
@@ -486,9 +504,9 @@
 
     def test_dict_of_dict(self):
         def f(n):
-            d = {}
+            d = self.newdict()
             d[5] = d
-            d[6] = {}
+            d[6] = self.newdict()
             return len(d[n])
 
         res = self.interpret(f, [5])
@@ -504,10 +522,9 @@
             pass
 
         def f(i):
-            d = {
-                A: 3,
-                B: 4,
-            }
+            d = self.newdict()
+            d[A] = 3
+            d[B] = 4
             if i:
                 cls = A
             else:
@@ -526,7 +543,9 @@
         class B(A):
             pass
 
-        d = {(A, 3): 3, (B, 0): 4}
+        d = self.newdict()
+        d[(A, 3)] = 3
+        d[(B, 0)] = 4
 
         def f(i):
             if i:
@@ -553,7 +572,9 @@
                 return 42
             return -1
         def g(n):
-            d = {1: n, 2: 2*n}
+            d = self.newdict()
+            d[1] = n
+            d[2] = 2*n
             return f(d)
         res = self.interpret(g, [3])
         assert res == 6
@@ -566,51 +587,19 @@
                 return 42

From noreply at buildbot.pypy.org  Tue Oct 29 23:10:48 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 23:10:48 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: some more test fixes
Message-ID: <20131029221048.C140F1C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67713:eb3bd8f08c9c
Date: 2013-10-29 18:10 -0400
http://bitbucket.org/pypy/pypy/changeset/eb3bd8f08c9c/

Log:	some more test fixes

diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -1,5 +1,7 @@
 from _numpypy.multiarray import *
 from _numpypy.umath import *
 
+True_ = dtype('bool').type(True)
+False_ = dtype('bool').type(False)
 newaxis = None
 ufunc = type(sin)
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -522,9 +522,9 @@
         assert str(exc.value) == \
             "could not broadcast input array from shape (2) into shape ()"
         a = array('abc')
-        assert str(a.real) == 'abc'
+        assert str(a.real) == str(a)
         # numpy imag for flexible types returns self
-        assert str(a.imag) == 'abc'
+        assert str(a.imag) == str(a)
         for t in 'complex64', 'complex128', 'clongdouble':
             complex_ = dtype(t).type
             O = complex(0, 0)

From noreply at buildbot.pypy.org  Tue Oct 29 23:42:31 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 23:42:31 +0100 (CET)
Subject: [pypy-commit] pypy default: test/fix segfault on bool_(x).round()
Message-ID: <20131029224231.CB3551C0163@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67714:66afd260d5e5
Date: 2013-10-29 18:31 -0400
http://bitbucket.org/pypy/pypy/changeset/66afd260d5e5/

Log:	test/fix segfault on bool_(x).round()

diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -23,19 +23,19 @@
         assert loads(dumps(sum(a))) == sum(a)
 
     def test_round(self):
-        from numpypy import int32, float64, complex128, bool
+        from numpypy import int32, float64, complex128, bool_
         i = int32(1337)
         f = float64(13.37)
         c = complex128(13 + 37.j)
-        b = bool(0)
+        b = bool_(1)
         assert i.round(decimals=-2) == 1300
         assert i.round(decimals=1) == 1337
         assert c.round() == c
         assert f.round() == 13.
         assert f.round(decimals=-1) == 10.
         assert f.round(decimals=1) == 13.4
-        exc = raises(AttributeError, 'b.round()')
-        assert exc.value[0] == "'bool' object has no attribute 'round'"
+        assert b.round() == 1.0
+        assert b.round(decimals=5) is b
 
     def test_attributes(self):
         import numpypy as np
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -400,6 +400,12 @@
             return 1
         return 0
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        if decimals != 0:
+            return v
+        return Float64().box(self.unbox(v))
+
 class Integer(Primitive):
     _mixin_ = True
 

From noreply at buildbot.pypy.org  Tue Oct 29 23:42:33 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 23:42:33 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: merge default
Message-ID: <20131029224233.0B2E51C0225@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67715:44bfbd81979e
Date: 2013-10-29 18:37 -0400
http://bitbucket.org/pypy/pypy/changeset/44bfbd81979e/

Log:	merge default

diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -23,19 +23,19 @@
         assert loads(dumps(sum(a))) == sum(a)
 
     def test_round(self):
-        from numpypy import int32, float64, complex128, bool
+        from numpypy import int32, float64, complex128, bool_
         i = int32(1337)
         f = float64(13.37)
         c = complex128(13 + 37.j)
-        b = bool(0)
+        b = bool_(1)
         assert i.round(decimals=-2) == 1300
         assert i.round(decimals=1) == 1337
         assert c.round() == c
         assert f.round() == 13.
         assert f.round(decimals=-1) == 10.
         assert f.round(decimals=1) == 13.4
-        exc = raises(AttributeError, 'b.round()')
-        assert exc.value[0] == "'bool' object has no attribute 'round'"
+        assert b.round() == 1.0
+        assert b.round(decimals=5) is b
 
     def test_attributes(self):
         import numpypy as np
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -400,6 +400,12 @@
             return 1
         return 0
 
+    @specialize.argtype(1)
+    def round(self, v, decimals=0):
+        if decimals != 0:
+            return v
+        return Float64().box(self.unbox(v))
+
 class Integer(Primitive):
     _mixin_ = True
 

From noreply at buildbot.pypy.org  Tue Oct 29 23:42:34 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 29 Oct 2013 23:42:34 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: port test_scalar
Message-ID: <20131029224234.39E6E1C0163@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67716:3f6b7f94ea32
Date: 2013-10-29 18:41 -0400
http://bitbucket.org/pypy/pypy/changeset/3f6b7f94ea32/

Log:	port test_scalar

diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -4,12 +4,16 @@
     spaceconfig = dict(usemodules=["micronumpy", "binascii", "struct"])
 
     def test_pickle(self):
-        from numpypy import dtype, int32, float64, complex128, zeros, sum
-        from numpypy.core.multiarray import scalar
+        from numpypy import dtype, zeros
+        try:
+            from numpy.core.multiarray import scalar
+        except ImportError:
+            # running on dummy module
+            from numpy import scalar
         from cPickle import loads, dumps
-        i = int32(1337)
-        f = float64(13.37)
-        c = complex128(13 + 37.j)
+        i = dtype('int32').type(1337)
+        f = dtype('float64').type(13.37)
+        c = dtype('complex128').type(13 + 37.j)
 
         assert i.__reduce__() == (scalar, (dtype('int32'), '9\x05\x00\x00'))
         assert f.__reduce__() == (scalar, (dtype('float64'), '=\n\xd7\xa3p\xbd*@'))
@@ -20,14 +24,14 @@
         assert loads(dumps(c)) == c
 
         a = zeros(3)
-        assert loads(dumps(sum(a))) == sum(a)
+        assert loads(dumps(a.sum())) == a.sum()
 
     def test_round(self):
-        from numpypy import int32, float64, complex128, bool_
-        i = int32(1337)
-        f = float64(13.37)
-        c = complex128(13 + 37.j)
-        b = bool_(1)
+        import numpy as np
+        i = np.dtype('int32').type(1337)
+        f = np.dtype('float64').type(13.37)
+        c = np.dtype('complex128').type(13 + 37.j)
+        b = np.dtype('bool').type(1)
         assert i.round(decimals=-2) == 1300
         assert i.round(decimals=1) == 1337
         assert c.round() == c
@@ -39,5 +43,6 @@
 
     def test_attributes(self):
         import numpypy as np
-        assert np.int64(0).dtype == np.dtype('int64')
-        assert np.int64(0).itemsize == 8
+        value = np.dtype('int64').type(12345)
+        assert value.dtype == np.dtype('int64')
+        assert value.itemsize == 8

From noreply at buildbot.pypy.org  Wed Oct 30 00:34:19 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 00:34:19 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: port test_subtype
Message-ID: <20131029233419.1F5511C0163@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67717:95d54735555f
Date: 2013-10-29 19:33 -0400
http://bitbucket.org/pypy/pypy/changeset/95d54735555f/

Log:	port test_subtype

diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -15,10 +15,10 @@
                     self.called_finalize = True
             return NoNew ''')
         cls.w_SubType = cls.space.appexec([], '''():
-            from numpypy import ndarray, asarray
+            from numpypy import ndarray, array
             class SubType(ndarray):
                 def __new__(obj, input_array):
-                    obj = asarray(input_array).view(obj)
+                    obj = array(input_array, copy=False).view(obj)
                     obj.called_new = True
                     return obj
                 def __array_finalize__(self, obj):
@@ -106,9 +106,9 @@
         assert not isinstance(b, self.NoNew)
 
     def test_sub_repeat(self):
-        from numpypy import repeat, array
+        from numpypy import array
         a = self.SubType(array([[1, 2], [3, 4]]))
-        b =  repeat(a, 3)
+        b = a.repeat(3)
         assert (b == [1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4]).all()
         assert isinstance(b, self.SubType)
 

From noreply at buildbot.pypy.org  Wed Oct 30 01:25:19 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 01:25:19 +0100 (CET)
Subject: [pypy-commit] pypy default: test/fix scalar construction default
	values
Message-ID: <20131030002519.2247F1C0225@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67718:26bd75c83407
Date: 2013-10-29 20:22 -0400
http://bitbucket.org/pypy/pypy/changeset/26bd75c83407/

Log:	test/fix scalar construction default values

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -34,7 +34,7 @@
         from pypy.module.micronumpy.interp_dtype import get_dtype_cache
         return get_dtype_cache(space).dtypes_by_name[name]
 
-    def new(space, w_subtype, w_value):
+    def new(space, w_subtype, w_value=None):
         dtype = _get_dtype(space)
         return dtype.itemtype.coerce_subtype(space, w_subtype, w_value)
 
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -3,6 +3,21 @@
 class AppTestScalar(BaseNumpyAppTest):
     spaceconfig = dict(usemodules=["micronumpy", "binascii", "struct"])
 
+    def test_init(self):
+        import numpypy as np
+        import math
+        assert np.intp() == np.intp(0)
+        assert np.intp('123') == np.intp(123)
+        raises(TypeError, np.intp, None)
+        assert np.float64() == np.float64(0)
+        assert math.isnan(np.float64(None))
+        assert np.bool_() == np.bool_(False)
+        assert np.bool_('abc') == np.bool_(True)
+        assert np.bool_(None) == np.bool_(False)
+        assert np.complex_() == np.complex_(0)
+        #raises(TypeError, np.complex_, '1+2j')
+        assert math.isnan(np.complex_(None))
+
     def test_pickle(self):
         from numpypy import dtype, int32, float64, complex128, zeros, sum
         from numpypy.core.multiarray import scalar
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -347,6 +347,8 @@
         return self._coerce(space, w_item)
 
     def _coerce(self, space, w_item):
+        if space.is_none(w_item):
+            return self.box(False)
         return self.box(space.is_true(w_item))
 
     def to_builtin_type(self, space, w_item):
@@ -410,6 +412,8 @@
     _mixin_ = True
 
     def _base_coerce(self, space, w_item):
+        if w_item is None:
+            return self.box(0)
         return self.box(space.int_w(space.call_function(space.w_int, w_item)))
     def _coerce(self, space, w_item):
         return self._base_coerce(space, w_item)
@@ -629,6 +633,8 @@
     _mixin_ = True
 
     def _coerce(self, space, w_item):
+        if w_item is None:
+            return self.box(0.0)
         if space.is_none(w_item):
             return self.box(rfloat.NAN)
         return self.box(space.float_w(space.call_function(space.w_float, w_item)))
@@ -999,6 +1005,10 @@
     _mixin_ = True
 
     def _coerce(self, space, w_item):
+        if w_item is None:
+            return self.box_complex(0.0, 0.0)
+        if space.is_none(w_item):
+            return self.box_complex(rfloat.NAN, rfloat.NAN)
         w_item = space.call_function(space.w_complex, w_item)
         real, imag = space.unpackcomplex(w_item)
         return self.box_complex(real, imag)

From noreply at buildbot.pypy.org  Wed Oct 30 02:50:26 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 02:50:26 +0100 (CET)
Subject: [pypy-commit] pypy default: fix some missing dtype aliases
Message-ID: <20131030015026.35EF01C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67719:9a9e3d3b03cc
Date: 2013-10-29 20:58 -0400
http://bitbucket.org/pypy/pypy/changeset/9a9e3d3b03cc/

Log:	fix some missing dtype aliases

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -491,6 +491,7 @@
             char=NPY_BOOLLTR,
             w_box_type=space.gettypefor(interp_boxes.W_BoolBox),
             alternate_constructors=[space.w_bool],
+            aliases=['bool8'],
         )
         self.w_int8dtype = W_Dtype(
             types.Int8(),
@@ -498,7 +499,8 @@
             kind=NPY_SIGNEDLTR,
             name="int8",
             char=NPY_BYTELTR,
-            w_box_type=space.gettypefor(interp_boxes.W_Int8Box)
+            w_box_type=space.gettypefor(interp_boxes.W_Int8Box),
+            aliases=['byte'],
         )
         self.w_uint8dtype = W_Dtype(
             types.UInt8(),
@@ -507,6 +509,7 @@
             name="uint8",
             char=NPY_UBYTELTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt8Box),
+            aliases=['ubyte'],
         )
         self.w_int16dtype = W_Dtype(
             types.Int16(),
@@ -515,6 +518,7 @@
             name="int16",
             char=NPY_SHORTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int16Box),
+            aliases=['short'],
         )
         self.w_uint16dtype = W_Dtype(
             types.UInt16(),
@@ -523,6 +527,7 @@
             name="uint16",
             char=NPY_USHORTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt16Box),
+            aliases=['ushort'],
         )
         self.w_int32dtype = W_Dtype(
             types.Int32(),
@@ -572,6 +577,7 @@
             char=NPY_LONGLONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
             alternate_constructors=[space.w_long],
+            aliases=['longlong'],
         )
         self.w_uint64dtype = W_Dtype(
             types.UInt64(),
@@ -580,6 +586,7 @@
             name="uint64",
             char=NPY_ULONGLONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt64Box),
+            aliases=['ulonglong'],
         )
         self.w_float32dtype = W_Dtype(
             types.Float32(),
@@ -588,6 +595,7 @@
             name="float32",
             char=NPY_FLOATLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Float32Box),
+            aliases=['single']
         )
         self.w_float64dtype = W_Dtype(
             types.Float64(),
@@ -617,6 +625,7 @@
             name="complex64",
             char=NPY_CFLOATLTR,
             w_box_type = space.gettypefor(interp_boxes.W_Complex64Box),
+            aliases=['csingle'],
             float_type = self.w_float32dtype,
         )
         self.w_complex128dtype = W_Dtype(
@@ -627,7 +636,7 @@
             char=NPY_CDOUBLELTR,
             w_box_type = space.gettypefor(interp_boxes.W_Complex128Box),
             alternate_constructors=[space.w_complex],
-            aliases=["complex"],
+            aliases=["complex", 'cfloat', 'cdouble'],
             float_type = self.w_float64dtype,
         )
         self.w_complexlongdtype = W_Dtype(
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -56,8 +56,21 @@
 
     def test_dtype_aliases(self):
         from numpypy import dtype
+        assert dtype('bool8') is dtype('bool')
+        assert dtype('byte') is dtype('int8')
+        assert dtype('ubyte') is dtype('uint8')
+        assert dtype('short') is dtype('int16')
+        assert dtype('ushort') is dtype('uint16')
+        assert dtype('longlong') is dtype('q')
+        assert dtype('ulonglong') is dtype('Q')
+        assert dtype("float") is dtype(float)
+        assert dtype('single') is dtype('float32')
+        assert dtype('double') is dtype('float64')
         assert dtype('longfloat').num in (12, 13)
         assert dtype('longdouble').num in (12, 13)
+        assert dtype('csingle') is dtype('complex64')
+        assert dtype('cfloat') is dtype('complex128')
+        assert dtype('cdouble') is dtype('complex128')
         assert dtype('clongfloat').num in (15, 16)
         assert dtype('clongdouble').num in (15, 16)
 
@@ -223,10 +236,6 @@
             pass
         assert True
 
-    def test_aliases(self):
-        from numpypy import dtype
-        assert dtype("float") is dtype(float)
-
     def test_index(self):
         import numpypy as np
         for dtype in [np.int8, np.int16, np.int32, np.int64]:

From noreply at buildbot.pypy.org  Wed Oct 30 02:50:27 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 02:50:27 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: merge default
Message-ID: <20131030015027.90A701C0163@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67720:86fae1ee9645
Date: 2013-10-29 21:34 -0400
http://bitbucket.org/pypy/pypy/changeset/86fae1ee9645/

Log:	merge default

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -34,7 +34,7 @@
         from pypy.module.micronumpy.interp_dtype import get_dtype_cache
         return get_dtype_cache(space).dtypes_by_name[name]
 
-    def new(space, w_subtype, w_value):
+    def new(space, w_subtype, w_value=None):
         dtype = _get_dtype(space)
         return dtype.itemtype.coerce_subtype(space, w_subtype, w_value)
 
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -491,6 +491,7 @@
             char=NPY_BOOLLTR,
             w_box_type=space.gettypefor(interp_boxes.W_BoolBox),
             alternate_constructors=[space.w_bool],
+            aliases=['bool8'],
         )
         self.w_int8dtype = W_Dtype(
             types.Int8(),
@@ -498,7 +499,8 @@
             kind=NPY_SIGNEDLTR,
             name="int8",
             char=NPY_BYTELTR,
-            w_box_type=space.gettypefor(interp_boxes.W_Int8Box)
+            w_box_type=space.gettypefor(interp_boxes.W_Int8Box),
+            aliases=['byte'],
         )
         self.w_uint8dtype = W_Dtype(
             types.UInt8(),
@@ -507,6 +509,7 @@
             name="uint8",
             char=NPY_UBYTELTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt8Box),
+            aliases=['ubyte'],
         )
         self.w_int16dtype = W_Dtype(
             types.Int16(),
@@ -515,6 +518,7 @@
             name="int16",
             char=NPY_SHORTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int16Box),
+            aliases=['short'],
         )
         self.w_uint16dtype = W_Dtype(
             types.UInt16(),
@@ -523,6 +527,7 @@
             name="uint16",
             char=NPY_USHORTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt16Box),
+            aliases=['ushort'],
         )
         self.w_int32dtype = W_Dtype(
             types.Int32(),
@@ -572,6 +577,7 @@
             char=NPY_LONGLONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
             alternate_constructors=[space.w_long],
+            aliases=['longlong'],
         )
         self.w_uint64dtype = W_Dtype(
             types.UInt64(),
@@ -580,6 +586,7 @@
             name="uint64",
             char=NPY_ULONGLONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt64Box),
+            aliases=['ulonglong'],
         )
         self.w_float32dtype = W_Dtype(
             types.Float32(),
@@ -588,6 +595,7 @@
             name="float32",
             char=NPY_FLOATLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Float32Box),
+            aliases=['single']
         )
         self.w_float64dtype = W_Dtype(
             types.Float64(),
@@ -617,6 +625,7 @@
             name="complex64",
             char=NPY_CFLOATLTR,
             w_box_type = space.gettypefor(interp_boxes.W_Complex64Box),
+            aliases=['csingle'],
             float_type = self.w_float32dtype,
         )
         self.w_complex128dtype = W_Dtype(
@@ -627,7 +636,7 @@
             char=NPY_CDOUBLELTR,
             w_box_type = space.gettypefor(interp_boxes.W_Complex128Box),
             alternate_constructors=[space.w_complex],
-            aliases=["complex"],
+            aliases=["complex", 'cfloat', 'cdouble'],
             float_type = self.w_float64dtype,
         )
         self.w_complexlongdtype = W_Dtype(
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -56,8 +56,21 @@
 
     def test_dtype_aliases(self):
         from numpypy import dtype
+        assert dtype('bool8') is dtype('bool')
+        assert dtype('byte') is dtype('int8')
+        assert dtype('ubyte') is dtype('uint8')
+        assert dtype('short') is dtype('int16')
+        assert dtype('ushort') is dtype('uint16')
+        assert dtype('longlong') is dtype('q')
+        assert dtype('ulonglong') is dtype('Q')
+        assert dtype("float") is dtype(float)
+        assert dtype('single') is dtype('float32')
+        assert dtype('double') is dtype('float64')
         assert dtype('longfloat').num in (12, 13)
         assert dtype('longdouble').num in (12, 13)
+        assert dtype('csingle') is dtype('complex64')
+        assert dtype('cfloat') is dtype('complex128')
+        assert dtype('cdouble') is dtype('complex128')
         assert dtype('clongfloat').num in (15, 16)
         assert dtype('clongdouble').num in (15, 16)
 
@@ -223,10 +236,6 @@
             pass
         assert True
 
-    def test_aliases(self):
-        from numpypy import dtype
-        assert dtype("float") is dtype(float)
-
     def test_index(self):
         import numpypy as np
         for dtype in [np.int8, np.int16, np.int32, np.int64]:
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -3,6 +3,21 @@
 class AppTestScalar(BaseNumpyAppTest):
     spaceconfig = dict(usemodules=["micronumpy", "binascii", "struct"])
 
+    def test_init(self):
+        import numpypy as np
+        import math
+        assert np.intp() == np.intp(0)
+        assert np.intp('123') == np.intp(123)
+        raises(TypeError, np.intp, None)
+        assert np.float64() == np.float64(0)
+        assert math.isnan(np.float64(None))
+        assert np.bool_() == np.bool_(False)
+        assert np.bool_('abc') == np.bool_(True)
+        assert np.bool_(None) == np.bool_(False)
+        assert np.complex_() == np.complex_(0)
+        #raises(TypeError, np.complex_, '1+2j')
+        assert math.isnan(np.complex_(None))
+
     def test_pickle(self):
         from numpypy import dtype, zeros
         try:
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -347,6 +347,8 @@
         return self._coerce(space, w_item)
 
     def _coerce(self, space, w_item):
+        if space.is_none(w_item):
+            return self.box(False)
         return self.box(space.is_true(w_item))
 
     def to_builtin_type(self, space, w_item):
@@ -410,6 +412,8 @@
     _mixin_ = True
 
     def _base_coerce(self, space, w_item):
+        if w_item is None:
+            return self.box(0)
         return self.box(space.int_w(space.call_function(space.w_int, w_item)))
     def _coerce(self, space, w_item):
         return self._base_coerce(space, w_item)
@@ -629,6 +633,8 @@
     _mixin_ = True
 
     def _coerce(self, space, w_item):
+        if w_item is None:
+            return self.box(0.0)
         if space.is_none(w_item):
             return self.box(rfloat.NAN)
         return self.box(space.float_w(space.call_function(space.w_float, w_item)))
@@ -999,6 +1005,10 @@
     _mixin_ = True
 
     def _coerce(self, space, w_item):
+        if w_item is None:
+            return self.box_complex(0.0, 0.0)
+        if space.is_none(w_item):
+            return self.box_complex(rfloat.NAN, rfloat.NAN)
         w_item = space.call_function(space.w_complex, w_item)
         real, imag = space.unpackcomplex(w_item)
         return self.box_complex(real, imag)

From noreply at buildbot.pypy.org  Wed Oct 30 02:50:28 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 02:50:28 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: add some types to the dummy test
	module
Message-ID: <20131030015028.C30BF1C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67721:ce531180fd41
Date: 2013-10-29 21:16 -0400
http://bitbucket.org/pypy/pypy/changeset/ce531180fd41/

Log:	add some types to the dummy test module

diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -1,7 +1,30 @@
 from _numpypy.multiarray import *
 from _numpypy.umath import *
 
-True_ = dtype('bool').type(True)
-False_ = dtype('bool').type(False)
 newaxis = None
 ufunc = type(sin)
+
+types = ['bool8', 'byte', 'ubyte', 'short', 'ushort', 'longlong', 'ulonglong',
+         'single', 'longfloat', 'longdouble', 'csingle', 'cfloat', 'void']
+for t in ('int', 'uint'):
+    for s in (8, 16, 32, 64, 'p'):
+        types.append(t + str(s))
+for s in (16, 32, 64):
+    types.append('float' + str(s))
+for s in (64, 128):
+    types.append('complex' + str(s))
+for t in types:
+    globals()[t] = dtype(t).type
+
+types = ['bool', 'int', 'float', 'complex', 'str', 'unicode']
+for t in types:
+    globals()[t + '_'] = dtype(t).type
+del types
+
+types = ['Generic', 'Number', 'Integer', 'SignedInteger', 'UnsignedInteger',
+         'Inexact', 'Floating', 'ComplexFloating', 'Character']
+for t in types:
+    globals()[t.lower()] = typeinfo[t]
+
+True_ = bool_(True)
+False_ = bool_(False)

From noreply at buildbot.pypy.org  Wed Oct 30 02:50:29 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 02:50:29 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: fix last failing test in
	micronumpy
Message-ID: <20131030015029.DE6F41C00EC@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67722:fb7ee14d8205
Date: 2013-10-29 21:43 -0400
http://bitbucket.org/pypy/pypy/changeset/fb7ee14d8205/

Log:	fix last failing test in micronumpy

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -937,7 +937,11 @@
 
     def test_typeinfo(self):
         from numpypy import void, number, int64, bool_, complex64, complex128, float16
-        from numpypy.core.multiarray import typeinfo
+        try:
+            from numpy.core.multiarray import typeinfo
+        except ImportError:
+            # running on dummy module
+            from numpypy import typeinfo
         assert typeinfo['Number'] == number
         assert typeinfo['LONGLONG'] == ('q', 9, 64, 8, 9223372036854775807L, -9223372036854775808L, int64)
         assert typeinfo['VOID'] == ('V', 20, 0, 1, void)

From noreply at buildbot.pypy.org  Wed Oct 30 03:33:57 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 03:33:57 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: remove lib_pypy/numpypy
Message-ID: <20131030023357.530FC1C0225@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67723:441bb5cabfb7
Date: 2013-10-29 21:44 -0400
http://bitbucket.org/pypy/pypy/changeset/441bb5cabfb7/

Log:	remove lib_pypy/numpypy

diff too long, truncating to 2000 out of 6319 lines

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from . import core
-from .core import *
-from . import lib
-from .lib import *
-
-from __builtin__ import bool, int, long, float, complex, object, unicode, str
-
-from .core import round, abs, max, min
-
-__version__ = '1.7.0'
-
-__all__ = ['__version__']
-__all__ += core.__all__
-__all__ += lib.__all__
-
-#import sys
-#sys.modules.setdefault('numpy', sys.modules['numpypy'])
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from __future__ import division, absolute_import, print_function
-
-from . import multiarray
-from . import umath
-from . import numeric
-from .numeric import *
-from . import fromnumeric
-from .fromnumeric import *
-from . import shape_base
-from .shape_base import *
-
-from .fromnumeric import amax as max, amin as min, \
-    round_ as round
-from .numeric import absolute as abs
-
-__all__ = []
-__all__ += numeric.__all__
-__all__ += fromnumeric.__all__
-__all__ += shape_base.__all__
diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/_methods.py
+++ /dev/null
@@ -1,124 +0,0 @@
-"""
-Array methods which are called by the both the C-code for the method
-and the Python code for the NumPy-namespace function
-
-"""
-from __future__ import division, absolute_import, print_function
-
-import warnings
-
-from . import multiarray as mu
-from . import umath as um
-from .numeric import asanyarray
-from . import numerictypes as nt
-
-def _amax(a, axis=None, out=None, keepdims=False):
-    return um.maximum.reduce(a, axis=axis,
-                            out=out, keepdims=keepdims)
-
-def _amin(a, axis=None, out=None, keepdims=False):
-    return um.minimum.reduce(a, axis=axis,
-                            out=out, keepdims=keepdims)
-
-def _sum(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.add.reduce(a, axis=axis, dtype=dtype,
-                            out=out, keepdims=keepdims)
-
-def _prod(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.multiply.reduce(a, axis=axis, dtype=dtype,
-                            out=out, keepdims=keepdims)
-
-def _any(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.logical_or.reduce(a, axis=axis, dtype=dtype, out=out,
-                                keepdims=keepdims)
-
-def _all(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.logical_and.reduce(a, axis=axis, dtype=dtype, out=out,
-                                 keepdims=keepdims)
-
-def _count_reduce_items(arr, axis):
-    if axis is None:
-        axis = tuple(range(arr.ndim))
-    if not isinstance(axis, tuple):
-        axis = (axis,)
-    items = 1
-    for ax in axis:
-        items *= arr.shape[ax]
-    return items
-
-def _mean(a, axis=None, dtype=None, out=None, keepdims=False):
-    arr = asanyarray(a)
-
-    rcount = _count_reduce_items(arr, axis)
-    # Make this warning show up first
-    if rcount == 0:
-        warnings.warn("Mean of empty slice.", RuntimeWarning)
-
-
-    # Cast bool, unsigned int, and int to float64 by default
-    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
-        dtype = mu.dtype('f8')
-
-    ret = um.add.reduce(arr, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
-    if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(
-                ret, rcount, out=ret, casting='unsafe', subok=False)
-    else:
-        ret = ret.dtype.type(ret / rcount)
-
-    return ret
-
-def _var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
-    arr = asanyarray(a)
-
-    rcount = _count_reduce_items(arr, axis)
-    # Make this warning show up on top.
-    if ddof >= rcount:
-        warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning)
-
-    # Cast bool, unsigned int, and int to float64 by default
-    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
-        dtype = mu.dtype('f8')
-
-    # Compute the mean.
-    # Note that if dtype is not of inexact type then arraymean will
-    # not be either.
-    arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True)
-    if isinstance(arrmean, mu.ndarray):
-        arrmean = um.true_divide(
-                arrmean, rcount, out=arrmean, casting='unsafe', subok=False)
-    else:
-        arrmean = arrmean.dtype.type(arrmean / rcount)
-
-    # Compute sum of squared deviations from mean
-    # Note that x may not be inexact and that we need it to be an array,
-    # not a scalar.
-    x = asanyarray(arr - arrmean)
-    if issubclass(arr.dtype.type, nt.complexfloating):
-        x = um.multiply(x, um.conjugate(x), out=x).real
-    else:
-        x = um.multiply(x, x, out=x)
-    ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
-
-    # Compute degrees of freedom and make sure it is not negative.
-    rcount = max([rcount - ddof, 0])
-
-    # divide by degrees of freedom
-    if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(
-                ret, rcount, out=ret, casting='unsafe', subok=False)
-    else:
-        ret = ret.dtype.type(ret / rcount)
-
-    return ret
-
-def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
-    ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
-               keepdims=keepdims)
-
-    if isinstance(ret, mu.ndarray):
-        ret = um.sqrt(ret, out=ret)
-    else:
-        ret = ret.dtype.type(um.sqrt(ret))
-
-    return ret
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ /dev/null
@@ -1,751 +0,0 @@
-"""Array printing function
-
-$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $
-"""
-__all__ = ["array2string", "set_printoptions", "get_printoptions"]
-__docformat__ = 'restructuredtext'
-
-#
-# Written by Konrad Hinsen <hinsenk at ere.umontreal.ca>
-# last revision: 1996-3-13
-# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details)
-# and by Perry Greenfield 2000-4-1 for numarray
-# and by Travis Oliphant  2005-8-22 for numpy
-
-import sys
-import numerictypes as _nt
-from umath import maximum, minimum, absolute, not_equal, isnan, isinf
-#from multiarray import format_longfloat, datetime_as_string, datetime_data
-from fromnumeric import ravel
-
-
-def product(x, y): return x*y
-
-_summaryEdgeItems = 3     # repr N leading and trailing items of each dimension
-_summaryThreshold = 1000 # total items > triggers array summarization
-
-_float_output_precision = 8
-_float_output_suppress_small = False
-_line_width = 75
-_nan_str = 'nan'
-_inf_str = 'inf'
-_formatter = None  # formatting function for array elements
-
-if sys.version_info[0] >= 3:
-    from functools import reduce
-
-def set_printoptions(precision=None, threshold=None, edgeitems=None,
-                     linewidth=None, suppress=None,
-                     nanstr=None, infstr=None,
-                     formatter=None):
-    """
-    Set printing options.
-
-    These options determine the way floating point numbers, arrays and
-    other NumPy objects are displayed.
-
-    Parameters
-    ----------
-    precision : int, optional
-        Number of digits of precision for floating point output (default 8).
-    threshold : int, optional
-        Total number of array elements which trigger summarization
-        rather than full repr (default 1000).
-    edgeitems : int, optional
-        Number of array items in summary at beginning and end of
-        each dimension (default 3).
-    linewidth : int, optional
-        The number of characters per line for the purpose of inserting
-        line breaks (default 75).
-    suppress : bool, optional
-        Whether or not suppress printing of small floating point values
-        using scientific notation (default False).
-    nanstr : str, optional
-        String representation of floating point not-a-number (default nan).
-    infstr : str, optional
-        String representation of floating point infinity (default inf).
-    formatter : dict of callables, optional
-        If not None, the keys should indicate the type(s) that the respective
-        formatting function applies to.  Callables should return a string.
-        Types that are not specified (by their corresponding keys) are handled
-        by the default formatters.  Individual types for which a formatter
-        can be set are::
-
-            - 'bool'
-            - 'int'
-            - 'timedelta' : a `numpy.timedelta64`
-            - 'datetime' : a `numpy.datetime64`
-            - 'float'
-            - 'longfloat' : 128-bit floats
-            - 'complexfloat'
-            - 'longcomplexfloat' : composed of two 128-bit floats
-            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
-            - 'str' : all other strings
-
-        Other keys that can be used to set a group of types at once are::
-
-            - 'all' : sets all types
-            - 'int_kind' : sets 'int'
-            - 'float_kind' : sets 'float' and 'longfloat'
-            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
-            - 'str_kind' : sets 'str' and 'numpystr'
-
-    See Also
-    --------
-    get_printoptions, set_string_function, array2string
-
-    Notes
-    -----
-    `formatter` is always reset with a call to `set_printoptions`.
-
-    Examples
-    --------
-    Floating point precision can be set:
-
-    >>> np.set_printoptions(precision=4)
-    >>> print np.array([1.123456789])
-    [ 1.1235]
-
-    Long arrays can be summarised:
-
-    >>> np.set_printoptions(threshold=5)
-    >>> print np.arange(10)
-    [0 1 2 ..., 7 8 9]
-
-    Small results can be suppressed:
-
-    >>> eps = np.finfo(float).eps
-    >>> x = np.arange(4.)
-    >>> x**2 - (x + eps)**2
-    array([ -4.9304e-32,  -4.4409e-16,   0.0000e+00,   0.0000e+00])
-    >>> np.set_printoptions(suppress=True)
-    >>> x**2 - (x + eps)**2
-    array([-0., -0.,  0.,  0.])
-
-    A custom formatter can be used to display array elements as desired:
-
-    >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)})
-    >>> x = np.arange(3)
-    >>> x
-    array([int: 0, int: -1, int: -2])
-    >>> np.set_printoptions()  # formatter gets reset
-    >>> x
-    array([0, 1, 2])
-
-    To put back the default options, you can use:
-
-    >>> np.set_printoptions(edgeitems=3,infstr='inf',
-    ... linewidth=75, nanstr='nan', precision=8,
-    ... suppress=False, threshold=1000, formatter=None)
-    """
-
-    global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \
-           _line_width, _float_output_suppress_small, _nan_str, _inf_str, \
-           _formatter
-    if linewidth is not None:
-        _line_width = linewidth
-    if threshold is not None:
-        _summaryThreshold = threshold
-    if edgeitems is not None:
-        _summaryEdgeItems = edgeitems
-    if precision is not None:
-        _float_output_precision = precision
-    if suppress is not None:
-        _float_output_suppress_small = not not suppress
-    if nanstr is not None:
-        _nan_str = nanstr
-    if infstr is not None:
-        _inf_str = infstr
-    _formatter = formatter
-
-def get_printoptions():
-    """
-    Return the current print options.
-
-    Returns
-    -------
-    print_opts : dict
-        Dictionary of current print options with keys
-
-          - precision : int
-          - threshold : int
-          - edgeitems : int
-          - linewidth : int
-          - suppress : bool
-          - nanstr : str
-          - infstr : str
-          - formatter : dict of callables
-
-        For a full description of these options, see `set_printoptions`.
-
-    See Also
-    --------
-    set_printoptions, set_string_function
-
-    """
-    d = dict(precision=_float_output_precision,
-             threshold=_summaryThreshold,
-             edgeitems=_summaryEdgeItems,
-             linewidth=_line_width,
-             suppress=_float_output_suppress_small,
-             nanstr=_nan_str,
-             infstr=_inf_str,
-             formatter=_formatter)
-    return d
-
-def _leading_trailing(a):
-    import numeric as _nc
-    if a.ndim == 1:
-        if len(a) > 2*_summaryEdgeItems:
-            b = _nc.concatenate((a[:_summaryEdgeItems],
-                                     a[-_summaryEdgeItems:]))
-        else:
-            b = a
-    else:
-        if len(a) > 2*_summaryEdgeItems:
-            l = [_leading_trailing(a[i]) for i in range(
-                min(len(a), _summaryEdgeItems))]
-            l.extend([_leading_trailing(a[-i]) for i in range(
-                min(len(a), _summaryEdgeItems),0,-1)])
-        else:
-            l = [_leading_trailing(a[i]) for i in range(0, len(a))]
-        b = _nc.concatenate(tuple(l))
-    return b
-
-def _boolFormatter(x):
-    if x:
-        return ' True'
-    else:
-        return 'False'
-
-
-def repr_format(x):
-    return repr(x)
-
-def _array2string(a, max_line_width, precision, suppress_small, separator=' ',
-                  prefix="", formatter=None):
-
-    if max_line_width is None:
-        max_line_width = _line_width
-
-    if precision is None:
-        precision = _float_output_precision
-
-    if suppress_small is None:
-        suppress_small = _float_output_suppress_small
-
-    if formatter is None:
-        formatter = _formatter
-
-    if a.size > _summaryThreshold:
-        summary_insert = "..., "
-        data = _leading_trailing(a)
-    else:
-        summary_insert = ""
-        data = ravel(a)
-
-    formatdict = {'bool' : _boolFormatter,
-                  'int' : IntegerFormat(data),
-                  'float' : FloatFormat(data, precision, suppress_small),
-                  'longfloat' : FloatFormat(data, precision, suppress_small),
-                  'complexfloat' : ComplexFormat(data, precision,
-                                                 suppress_small),
-                  'longcomplexfloat' : ComplexFormat(data, precision,
-                                                     suppress_small),
-                  'datetime' : DatetimeFormat(data),
-                  'timedelta' : TimedeltaFormat(data),
-                  'numpystr' : repr_format,
-                  'str' : str}
-
-    if formatter is not None:
-        fkeys = [k for k in formatter.keys() if formatter[k] is not None]
-        if 'all' in fkeys:
-            for key in formatdict.keys():
-                formatdict[key] = formatter['all']
-        if 'int_kind' in fkeys:
-            for key in ['int']:
-                formatdict[key] = formatter['int_kind']
-        if 'float_kind' in fkeys:
-            for key in ['float', 'longfloat']:
-                formatdict[key] = formatter['float_kind']
-        if 'complex_kind' in fkeys:
-            for key in ['complexfloat', 'longcomplexfloat']:
-                formatdict[key] = formatter['complex_kind']
-        if 'str_kind' in fkeys:
-            for key in ['numpystr', 'str']:
-                formatdict[key] = formatter['str_kind']
-        for key in formatdict.keys():
-            if key in fkeys:
-                formatdict[key] = formatter[key]
-
-    try:
-        format_function = a._format
-        msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \
-              "will be removed in 2.1. Use the `formatter` kw instead."
-        import warnings
-        warnings.warn(msg, DeprecationWarning)
-    except AttributeError:
-        # find the right formatting function for the array
-        dtypeobj = a.dtype.type
-        if issubclass(dtypeobj, _nt.bool_):
-            format_function = formatdict['bool']
-        elif issubclass(dtypeobj, _nt.integer):
-            #if issubclass(dtypeobj, _nt.timedelta64):
-            #    format_function = formatdict['timedelta']
-            #else:
-            format_function = formatdict['int']
-        elif issubclass(dtypeobj, _nt.floating):
-            if hasattr(_nt, 'longfloat') and issubclass(dtypeobj, _nt.longfloat):
-                format_function = formatdict['longfloat']
-            else:
-                format_function = formatdict['float']
-        elif issubclass(dtypeobj, _nt.complexfloating):
-            if hasattr(_nt, 'clongfloat') and issubclass(dtypeobj, _nt.clongfloat):
-                format_function = formatdict['longcomplexfloat']
-            else:
-                format_function = formatdict['complexfloat']
-        elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)):
-            format_function = formatdict['numpystr']
-        #elif issubclass(dtypeobj, _nt.datetime64):
-        #    format_function = formatdict['datetime']
-        else:
-            format_function = formatdict['str']
-
-    # skip over "["
-    next_line_prefix = " "
-    # skip over array(
-    next_line_prefix += " "*len(prefix)
-
-    lst = _formatArray(a, format_function, len(a.shape), max_line_width,
-                       next_line_prefix, separator,
-                       _summaryEdgeItems, summary_insert)[:-1]
-    return lst
-
-def _convert_arrays(obj):
-    import numeric as _nc
-    newtup = []
-    for k in obj:
-        if isinstance(k, _nc.ndarray):
-            k = k.tolist()
-        elif isinstance(k, tuple):
-            k = _convert_arrays(k)
-        newtup.append(k)
-    return tuple(newtup)
-
-
-def array2string(a, max_line_width=None, precision=None,
-                 suppress_small=None, separator=' ', prefix="",
-                 style=repr, formatter=None):
-    """
-    Return a string representation of an array.
-
-    Parameters
-    ----------
-    a : ndarray
-        Input array.
-    max_line_width : int, optional
-        The maximum number of columns the string should span. Newline
-        characters splits the string appropriately after array elements.
-    precision : int, optional
-        Floating point precision. Default is the current printing
-        precision (usually 8), which can be altered using `set_printoptions`.
-    suppress_small : bool, optional
-        Represent very small numbers as zero. A number is "very small" if it
-        is smaller than the current printing precision.
-    separator : str, optional
-        Inserted between elements.
-    prefix : str, optional
-        An array is typically printed as::
-
-          'prefix(' + array2string(a) + ')'
-
-        The length of the prefix string is used to align the
-        output correctly.
-    style : function, optional
-        A function that accepts an ndarray and returns a string.  Used only
-        when the shape of `a` is equal to ``()``, i.e. for 0-D arrays.
-    formatter : dict of callables, optional
-        If not None, the keys should indicate the type(s) that the respective
-        formatting function applies to.  Callables should return a string.
-        Types that are not specified (by their corresponding keys) are handled
-        by the default formatters.  Individual types for which a formatter
-        can be set are::
-
-            - 'bool'
-            - 'int'
-            - 'timedelta' : a `numpy.timedelta64`
-            - 'datetime' : a `numpy.datetime64`
-            - 'float'
-            - 'longfloat' : 128-bit floats
-            - 'complexfloat'
-            - 'longcomplexfloat' : composed of two 128-bit floats
-            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
-            - 'str' : all other strings
-
-        Other keys that can be used to set a group of types at once are::
-
-            - 'all' : sets all types
-            - 'int_kind' : sets 'int'
-            - 'float_kind' : sets 'float' and 'longfloat'
-            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
-            - 'str_kind' : sets 'str' and 'numpystr'
-
-    Returns
-    -------
-    array_str : str
-        String representation of the array.
-
-    Raises
-    ------
-    TypeError : if a callable in `formatter` does not return a string.
-
-    See Also
-    --------
-    array_str, array_repr, set_printoptions, get_printoptions
-
-    Notes
-    -----
-    If a formatter is specified for a certain type, the `precision` keyword is
-    ignored for that type.
-
-    Examples
-    --------
-    >>> x = np.array([1e-16,1,2,3])
-    >>> print np.array2string(x, precision=2, separator=',',
-    ...                       suppress_small=True)
-    [ 0., 1., 2., 3.]
-
-    >>> x  = np.arange(3.)
-    >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x})
-    '[0.00 1.00 2.00]'
-
-    >>> x  = np.arange(3)
-    >>> np.array2string(x, formatter={'int':lambda x: hex(x)})
-    '[0x0L 0x1L 0x2L]'
-
-    """
-
-    if a.shape == ():
-        x = a.item()
-        try:
-            lst = a._format(x)
-            msg = "The `_format` attribute is deprecated in Numpy " \
-                  "2.0 and will be removed in 2.1. Use the " \
-                  "`formatter` kw instead."
-            import warnings
-            warnings.warn(msg, DeprecationWarning)
-        except AttributeError:
-            if isinstance(x, tuple):
-                x = _convert_arrays(x)
-            lst = style(x)
-    elif reduce(product, a.shape) == 0:
-        # treat as a null array if any of shape elements == 0
-        lst = "[]"
-    else:
-        lst = _array2string(a, max_line_width, precision, suppress_small,
-                            separator, prefix, formatter=formatter)
-    return lst
-
-def _extendLine(s, line, word, max_line_len, next_line_prefix):
-    if len(line.rstrip()) + len(word.rstrip()) >= max_line_len:
-        s += line.rstrip() + "\n"
-        line = next_line_prefix
-    line += word
-    return s, line
-
-
-def _formatArray(a, format_function, rank, max_line_len,
-                 next_line_prefix, separator, edge_items, summary_insert):
-    """formatArray is designed for two modes of operation:
-
-    1. Full output
-
-    2. Summarized output
-
-    """
-    if rank == 0:
-        obj = a.item()
-        if isinstance(obj, tuple):
-            obj = _convert_arrays(obj)
-        return str(obj)
-
-    if summary_insert and 2*edge_items < len(a):
-        leading_items, trailing_items, summary_insert1 = \
-                       edge_items, edge_items, summary_insert
-    else:
-        leading_items, trailing_items, summary_insert1 = 0, len(a), ""
-
-    if rank == 1:
-        s = ""
-        line = next_line_prefix
-        for i in xrange(leading_items):
-            word = format_function(a[i]) + separator
-            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-
-        if summary_insert1:
-            s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix)
-
-        for i in xrange(trailing_items, 1, -1):
-            word = format_function(a[-i]) + separator
-            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-
-        word = format_function(a[-1])
-        s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-        s += line + "]\n"
-        s = '[' + s[len(next_line_prefix):]
-    else:
-        s = '['
-        sep = separator.rstrip()
-        for i in xrange(leading_items):
-            if i > 0:
-                s += next_line_prefix
-            s += _formatArray(a[i], format_function, rank-1, max_line_len,
-                              " " + next_line_prefix, separator, edge_items,
-                              summary_insert)
-            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
-
-        if summary_insert1:
-            s += next_line_prefix + summary_insert1 + "\n"
-
-        for i in xrange(trailing_items, 1, -1):
-            if leading_items or i != trailing_items:
-                s += next_line_prefix
-            s += _formatArray(a[-i], format_function, rank-1, max_line_len,
-                              " " + next_line_prefix, separator, edge_items,
-                              summary_insert)
-            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
-        if leading_items or trailing_items > 1:
-            s += next_line_prefix
-        s += _formatArray(a[-1], format_function, rank-1, max_line_len,
-                          " " + next_line_prefix, separator, edge_items,
-                          summary_insert).rstrip()+']\n'
-    return s
-
-class FloatFormat(object):
-    def __init__(self, data, precision, suppress_small, sign=False):
-        self.precision = precision
-        self.suppress_small = suppress_small
-        self.sign = sign
-        self.exp_format = False
-        self.large_exponent = False
-        self.max_str_len = 0
-        try:
-            self.fillFormat(data)
-        except (TypeError, NotImplementedError):
-            # if reduce(data) fails, this instance will not be called, just
-            # instantiated in formatdict.
-            pass
-
-    def fillFormat(self, data):
-        import numeric as _nc
-        errstate = _nc.seterr(all='ignore')
-        try:
-            special = isnan(data) | isinf(data)
-            valid = not_equal(data, 0) & ~special
-            non_zero = absolute(data.compress(valid))
-            if len(non_zero) == 0:
-                max_val = 0.
-                min_val = 0.
-            else:
-                max_val = maximum.reduce(non_zero)
-                min_val = minimum.reduce(non_zero)
-                if max_val >= 1.e8:
-                    self.exp_format = True
-                if not self.suppress_small and (min_val < 0.0001
-                                           or max_val/min_val > 1000.):
-                    self.exp_format = True
-        finally:
-            _nc.seterr(**errstate)
-
-        if self.exp_format:
-            self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100
-            self.max_str_len = 8 + self.precision
-            if self.large_exponent:
-                self.max_str_len += 1
-            if self.sign:
-                format = '%+'
-            else:
-                format = '%'
-            format = format + '%d.%de' % (self.max_str_len, self.precision)
-        else:
-            format = '%%.%df' % (self.precision,)
-            if len(non_zero):
-                precision = max([_digits(x, self.precision, format)
-                                 for x in non_zero])
-            else:
-                precision = 0
-            precision = min(self.precision, precision)
-            self.max_str_len = len(str(int(max_val))) + precision + 2
-            if _nc.any(special):
-                self.max_str_len = max(self.max_str_len,
-                                       len(_nan_str),
-                                       len(_inf_str)+1)
-            if self.sign:
-                format = '%#+'
-            else:
-                format = '%#'
-            format = format + '%d.%df' % (self.max_str_len, precision)
-
-        self.special_fmt = '%%%ds' % (self.max_str_len,)
-        self.format = format
-
-    def __call__(self, x, strip_zeros=True):
-        import numeric as _nc
-        err = _nc.seterr(invalid='ignore')
-        try:
-            if isnan(x):
-                if self.sign:
-                    return self.special_fmt % ('+' + _nan_str,)
-                else:
-                    return self.special_fmt % (_nan_str,)
-            elif isinf(x):
-                if x > 0:
-                    if self.sign:
-                        return self.special_fmt % ('+' + _inf_str,)
-                    else:
-                        return self.special_fmt % (_inf_str,)
-                else:
-                    return self.special_fmt % ('-' + _inf_str,)
-        finally:
-            _nc.seterr(**err)
-
-        s = self.format % x
-        if self.large_exponent:
-            # 3-digit exponent
-            expsign = s[-3]
-            if expsign == '+' or expsign == '-':
-                s = s[1:-2] + '0' + s[-2:]
-        elif self.exp_format:
-            # 2-digit exponent
-            if s[-3] == '0':
-                s = ' ' + s[:-3] + s[-2:]
-        elif strip_zeros:
-            z = s.rstrip('0')
-            s = z + ' '*(len(s)-len(z))
-        return s
-
-
-def _digits(x, precision, format):
-    s = format % x
-    z = s.rstrip('0')
-    return precision - len(s) + len(z)
-
-
-_MAXINT = sys.maxint
-_MININT = -sys.maxint-1
-class IntegerFormat(object):
-    def __init__(self, data):
-        try:
-            max_str_len = max(len(str(maximum.reduce(data))),
-                              len(str(minimum.reduce(data))))
-            self.format = '%' + str(max_str_len) + 'd'
-        except (TypeError, NotImplementedError):
-            # if reduce(data) fails, this instance will not be called, just
-            # instantiated in formatdict.
-            pass
-        except ValueError:
-            # this occurs when everything is NA
-            pass
-
-    def __call__(self, x):
-        if _MININT < x < _MAXINT:
-            return self.format % x
-        else:
-            return "%s" % x
-
-class LongFloatFormat(object):
-    # XXX Have to add something to determine the width to use a la FloatFormat
-    # Right now, things won't line up properly
-    def __init__(self, precision, sign=False):
-        self.precision = precision
-        self.sign = sign
-
-    def __call__(self, x):
-        if isnan(x):
-            if self.sign:
-                return '+' + _nan_str
-            else:
-                return ' ' + _nan_str
-        elif isinf(x):
-            if x > 0:
-                if self.sign:
-                    return '+' + _inf_str
-                else:
-                    return ' ' + _inf_str
-            else:
-                return '-' + _inf_str
-        elif x >= 0:
-            if self.sign:
-                return '+' + format_longfloat(x, self.precision)
-            else:
-                return ' ' + format_longfloat(x, self.precision)
-        else:
-            return format_longfloat(x, self.precision)
-
-
-class LongComplexFormat(object):
-    def __init__(self, precision):
-        self.real_format = LongFloatFormat(precision)
-        self.imag_format = LongFloatFormat(precision, sign=True)
-
-    def __call__(self, x):
-        r = self.real_format(x.real)
-        i = self.imag_format(x.imag)
-        return r + i + 'j'
-
-
-class ComplexFormat(object):
-    def __init__(self, x, precision, suppress_small):
-        self.real_format = FloatFormat(x.real, precision, suppress_small)
-        self.imag_format = FloatFormat(x.imag, precision, suppress_small,
-                                       sign=True)
-
-    def __call__(self, x):
-        r = self.real_format(x.real, strip_zeros=False)
-        i = self.imag_format(x.imag, strip_zeros=False)
-        if not self.imag_format.exp_format:
-            z = i.rstrip('0')
-            i = z + 'j' + ' '*(len(i)-len(z))
-        else:
-            i = i + 'j'
-        return r + i
-
-class DatetimeFormat(object):
-    def __init__(self, x, unit=None,
-                timezone=None, casting='same_kind'):
-        # Get the unit from the dtype
-        if unit is None:
-            if x.dtype.kind == 'M':
-                unit = datetime_data(x.dtype)[0]
-            else:
-                unit = 's'
-
-        # If timezone is default, make it 'local' or 'UTC' based on the unit
-        if timezone is None:
-            # Date units -> UTC, time units -> local
-            if unit in ('Y', 'M', 'W', 'D'):
-                self.timezone = 'UTC'
-            else:
-                self.timezone = 'local'
-        else:
-            self.timezone = timezone
-        self.unit = unit
-        self.casting = casting
-
-    def __call__(self, x):
-        return "'%s'" % datetime_as_string(x,
-                                    unit=self.unit,
-                                    timezone=self.timezone,
-                                    casting=self.casting)
-
-class TimedeltaFormat(object):
-    def __init__(self, data):
-        if data.dtype.kind == 'm':
-            v = data.view('i8')
-            max_str_len = max(len(str(maximum.reduce(v))),
-                              len(str(minimum.reduce(v))))
-            self.format = '%' + str(max_str_len) + 'd'
-
-    def __call__(self, x):
-        return self.format % x.astype('i8')
-
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ /dev/null
@@ -1,2924 +0,0 @@
-######################################################################
-# This is a copy of numpy/core/fromnumeric.py modified for numpypy
-######################################################################
-"""Module containing non-deprecated functions borrowed from Numeric.
-
-"""
-from __future__ import division, absolute_import, print_function
-
-import types
-
-from . import multiarray as mu
-from . import umath as um
-from . import numerictypes as nt
-from .numeric import asarray, array, asanyarray, concatenate
-from . import _methods
-
-
-# functions that are methods
-__all__ = [
-        'alen', 'all', 'alltrue', 'amax', 'amin', 'any', 'argmax',
-        'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip',
-        'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean',
-        'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put',
-        'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_',
-        'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze',
-        'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var',
-        ]
-
-
-try:
-    _gentype = types.GeneratorType
-except AttributeError:
-    _gentype = type(None)
-
-# save away Python sum
-_sum_ = sum
-
-# functions that are now methods
-def _wrapit(obj, method, *args, **kwds):
-    try:
-        wrap = obj.__array_wrap__
-    except AttributeError:
-        wrap = None
-    result = getattr(asarray(obj), method)(*args, **kwds)
-    if wrap:
-        if not isinstance(result, mu.ndarray):
-            result = asarray(result)
-        result = wrap(result)
-    return result
-
-
-def take(a, indices, axis=None, out=None, mode='raise'):
-    """
-    Take elements from an array along an axis.
-
-    This function does the same thing as "fancy" indexing (indexing arrays
-    using arrays); however, it can be easier to use if you need elements
-    along a given axis.
-
-    Parameters
-    ----------
-    a : array_like
-        The source array.
-    indices : array_like
-        The indices of the values to extract.
-
-        .. versionadded:: 1.8.0
-
-        Also allow scalars for indices.
-    axis : int, optional
-        The axis over which to select values. By default, the flattened
-        input array is used.
-    out : ndarray, optional
-        If provided, the result will be placed in this array. It should
-        be of the appropriate shape and dtype.
-    mode : {'raise', 'wrap', 'clip'}, optional
-        Specifies how out-of-bounds indices will behave.
-
-        * 'raise' -- raise an error (default)
-        * 'wrap' -- wrap around
-        * 'clip' -- clip to the range
-
-        'clip' mode means that all indices that are too large are replaced
-        by the index that addresses the last element along that axis. Note
-        that this disables indexing with negative numbers.
-
-    Returns
-    -------
-    subarray : ndarray
-        The returned array has the same type as `a`.
-
-    See Also
-    --------
-    ndarray.take : equivalent method
-
-    Examples
-    --------
-    >>> a = [4, 3, 5, 7, 6, 8]
-    >>> indices = [0, 1, 4]
-    >>> np.take(a, indices)
-    array([4, 3, 6])
-
-    In this example if `a` is an ndarray, "fancy" indexing can be used.
-
-    >>> a = np.array(a)
-    >>> a[indices]
-    array([4, 3, 6])
-
-    If `indices` is not one dimensional, the output also has these dimensions.
-
-    >>> np.take(a, [[0, 1], [2, 3]])
-    array([[4, 3],
-           [5, 7]])
-    """
-    try:
-        take = a.take
-    except AttributeError:
-        return _wrapit(a, 'take', indices, axis, out, mode)
-    return take(indices, axis, out, mode)
-
-
-# not deprecated --- copy if necessary, view otherwise
-def reshape(a, newshape, order='C'):
-    """
-    Gives a new shape to an array without changing its data.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be reshaped.
-    newshape : int or tuple of ints
-        The new shape should be compatible with the original shape. If
-        an integer, then the result will be a 1-D array of that length.
-        One shape dimension can be -1. In this case, the value is inferred
-        from the length of the array and remaining dimensions.
-    order : {'C', 'F', 'A'}, optional
-        Read the elements of `a` using this index order, and place the elements
-        into the reshaped array using this index order.  'C' means to
-        read / write the elements using C-like index order, with the last axis index
-        changing fastest, back to the first axis index changing slowest.  'F'
-        means to read / write the elements using Fortran-like index order, with
-        the first index changing fastest, and the last index changing slowest.
-        Note that the 'C' and 'F' options take no account of the memory layout
-        of the underlying array, and only refer to the order of indexing.  'A'
-        means to read / write the elements in Fortran-like index order if `a` is
-        Fortran *contiguous* in memory, C-like order otherwise.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        This will be a new view object if possible; otherwise, it will
-        be a copy.  Note there is no guarantee of the *memory layout* (C- or
-        Fortran- contiguous) of the returned array.
-
-    See Also
-    --------
-    ndarray.reshape : Equivalent method.
-
-    Notes
-    -----
-    It is not always possible to change the shape of an array without
-    copying the data. If you want an error to be raise if the data is copied,
-    you should assign the new shape to the shape attribute of the array::
-
-     >>> a = np.zeros((10, 2))
-     # A transpose make the array non-contiguous
-     >>> b = a.T
-     # Taking a view makes it possible to modify the shape without modifying the
-     # initial object.
-     >>> c = b.view()
-     >>> c.shape = (20)
-     AttributeError: incompatible shape for a non-contiguous array
-
-    The `order` keyword gives the index ordering both for *fetching* the values
-    from `a`, and then *placing* the values into the output array.  For example,
-    let's say you have an array:
-
-    >>> a = np.arange(6).reshape((3, 2))
-    >>> a
-    array([[0, 1],
-           [2, 3],
-           [4, 5]])
-
-    You can think of reshaping as first raveling the array (using the given
-    index order), then inserting the elements from the raveled array into the
-    new array using the same kind of index ordering as was used for the
-    raveling.
-
-    >>> np.reshape(a, (2, 3)) # C-like index ordering
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.reshape(a, (2, 3), order='F') # Fortran-like index ordering
-    array([[0, 4, 3],
-           [2, 1, 5]])
-    >>> np.reshape(np.ravel(a, order='F'), (2, 3), order='F')
-    array([[0, 4, 3],
-           [2, 1, 5]])
-
-    Examples
-    --------
-    >>> a = np.array([[1,2,3], [4,5,6]])
-    >>> np.reshape(a, 6)
-    array([1, 2, 3, 4, 5, 6])
-    >>> np.reshape(a, 6, order='F')
-    array([1, 4, 2, 5, 3, 6])
-
-    >>> np.reshape(a, (3,-1))       # the unspecified value is inferred to be 2
-    array([[1, 2],
-           [3, 4],
-           [5, 6]])
-    """
-    assert order == 'C'
-    try:
-        reshape = a.reshape
-    except AttributeError:
-        return _wrapit(a, 'reshape', newshape)
-    return reshape(newshape)
-
-
-def choose(a, choices, out=None, mode='raise'):
-    """
-    Construct an array from an index array and a set of arrays to choose from.
-
-    First of all, if confused or uncertain, definitely look at the Examples -
-    in its full generality, this function is less simple than it might
-    seem from the following code description (below ndi =
-    `numpy.lib.index_tricks`):
-
-    ``np.choose(a,c) == np.array([c[a[I]][I] for I in ndi.ndindex(a.shape)])``.
-
-    But this omits some subtleties.  Here is a fully general summary:
-
-    Given an "index" array (`a`) of integers and a sequence of `n` arrays
-    (`choices`), `a` and each choice array are first broadcast, as necessary,
-    to arrays of a common shape; calling these *Ba* and *Bchoices[i], i =
-    0,...,n-1* we have that, necessarily, ``Ba.shape == Bchoices[i].shape``
-    for each `i`.  Then, a new array with shape ``Ba.shape`` is created as
-    follows:
-
-    * if ``mode=raise`` (the default), then, first of all, each element of
-      `a` (and thus `Ba`) must be in the range `[0, n-1]`; now, suppose that
-      `i` (in that range) is the value at the `(j0, j1, ..., jm)` position
-      in `Ba` - then the value at the same position in the new array is the
-      value in `Bchoices[i]` at that same position;
-
-    * if ``mode=wrap``, values in `a` (and thus `Ba`) may be any (signed)
-      integer; modular arithmetic is used to map integers outside the range
-      `[0, n-1]` back into that range; and then the new array is constructed
-      as above;
-
-    * if ``mode=clip``, values in `a` (and thus `Ba`) may be any (signed)
-      integer; negative integers are mapped to 0; values greater than `n-1`
-      are mapped to `n-1`; and then the new array is constructed as above.
-
-    Parameters
-    ----------
-    a : int array
-        This array must contain integers in `[0, n-1]`, where `n` is the number
-        of choices, unless ``mode=wrap`` or ``mode=clip``, in which cases any
-        integers are permissible.
-    choices : sequence of arrays
-        Choice arrays. `a` and all of the choices must be broadcastable to the
-        same shape.  If `choices` is itself an array (not recommended), then
-        its outermost dimension (i.e., the one corresponding to
-        ``choices.shape[0]``) is taken as defining the "sequence".
-    out : array, optional
-        If provided, the result will be inserted into this array. It should
-        be of the appropriate shape and dtype.
-    mode : {'raise' (default), 'wrap', 'clip'}, optional
-        Specifies how indices outside `[0, n-1]` will be treated:
-
-          * 'raise' : an exception is raised
-          * 'wrap' : value becomes value mod `n`
-          * 'clip' : values < 0 are mapped to 0, values > n-1 are mapped to n-1
-
-    Returns
-    -------
-    merged_array : array
-        The merged result.
-
-    Raises
-    ------
-    ValueError: shape mismatch
-        If `a` and each choice array are not all broadcastable to the same
-        shape.
-
-    See Also
-    --------
-    ndarray.choose : equivalent method
-
-    Notes
-    -----
-    To reduce the chance of misinterpretation, even though the following
-    "abuse" is nominally supported, `choices` should neither be, nor be
-    thought of as, a single array, i.e., the outermost sequence-like container
-    should be either a list or a tuple.
-
-    Examples
-    --------
-
-    >>> choices = [[0, 1, 2, 3], [10, 11, 12, 13],
-    ...   [20, 21, 22, 23], [30, 31, 32, 33]]
-    >>> np.choose([2, 3, 1, 0], choices
-    ... # the first element of the result will be the first element of the
-    ... # third (2+1) "array" in choices, namely, 20; the second element
-    ... # will be the second element of the fourth (3+1) choice array, i.e.,
-    ... # 31, etc.
-    ... )
-    array([20, 31, 12,  3])
-    >>> np.choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1)
-    array([20, 31, 12,  3])
-    >>> # because there are 4 choice arrays
-    >>> np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4)
-    array([20,  1, 12,  3])
-    >>> # i.e., 0
-
-    A couple examples illustrating how choose broadcasts:
-
-    >>> a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]]
-    >>> choices = [-10, 10]
-    >>> np.choose(a, choices)
-    array([[ 10, -10,  10],
-           [-10,  10, -10],
-           [ 10, -10,  10]])
-
-    >>> # With thanks to Anne Archibald
-    >>> a = np.array([0, 1]).reshape((2,1,1))
-    >>> c1 = np.array([1, 2, 3]).reshape((1,3,1))
-    >>> c2 = np.array([-1, -2, -3, -4, -5]).reshape((1,1,5))
-    >>> np.choose(a, (c1, c2)) # result is 2x3x5, res[0,:,:]=c1, res[1,:,:]=c2
-    array([[[ 1,  1,  1,  1,  1],
-            [ 2,  2,  2,  2,  2],
-            [ 3,  3,  3,  3,  3]],
-           [[-1, -2, -3, -4, -5],
-            [-1, -2, -3, -4, -5],
-            [-1, -2, -3, -4, -5]]])
-
-    """
-    try:
-        choose = a.choose
-    except AttributeError:
-        return _wrapit(a, 'choose', choices, out=out, mode=mode)
-    return choose(choices, out=out, mode=mode)
-
-
-def repeat(a, repeats, axis=None):
-    """
-    Repeat elements of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    repeats : {int, array of ints}
-        The number of repetitions for each element.  `repeats` is broadcasted
-        to fit the shape of the given axis.
-    axis : int, optional
-        The axis along which to repeat values.  By default, use the
-        flattened input array, and return a flat output array.
-
-    Returns
-    -------
-    repeated_array : ndarray
-        Output array which has the same shape as `a`, except along
-        the given axis.
-
-    See Also
-    --------
-    tile : Tile an array.
-
-    Examples
-    --------
-    >>> x = np.array([[1,2],[3,4]])
-    >>> np.repeat(x, 2)
-    array([1, 1, 2, 2, 3, 3, 4, 4])
-    >>> np.repeat(x, 3, axis=1)
-    array([[1, 1, 1, 2, 2, 2],
-           [3, 3, 3, 4, 4, 4]])
-    >>> np.repeat(x, [1, 2], axis=0)
-    array([[1, 2],
-           [3, 4],
-           [3, 4]])
-
-    """
-    try:
-        repeat = a.repeat
-    except AttributeError:
-        return _wrapit(a, 'repeat', repeats, axis)
-    return repeat(repeats, axis)
-
-
-def put(a, ind, v, mode='raise'):
-    """
-    Replaces specified elements of an array with given values.
-
-    The indexing works on the flattened target array. `put` is roughly
-    equivalent to:
-
-    ::
-
-        a.flat[ind] = v
-
-    Parameters
-    ----------
-    a : ndarray
-        Target array.
-    ind : array_like
-        Target indices, interpreted as integers.
-    v : array_like
-        Values to place in `a` at target indices. If `v` is shorter than
-        `ind` it will be repeated as necessary.
-    mode : {'raise', 'wrap', 'clip'}, optional
-        Specifies how out-of-bounds indices will behave.
-
-        * 'raise' -- raise an error (default)
-        * 'wrap' -- wrap around
-        * 'clip' -- clip to the range
-
-        'clip' mode means that all indices that are too large are replaced
-        by the index that addresses the last element along that axis. Note
-        that this disables indexing with negative numbers.
-
-    See Also
-    --------
-    putmask, place
-
-    Examples
-    --------
-    >>> a = np.arange(5)
-    >>> np.put(a, [0, 2], [-44, -55])
-    >>> a
-    array([-44,   1, -55,   3,   4])
-
-    >>> a = np.arange(5)
-    >>> np.put(a, 22, -5, mode='clip')
-    >>> a
-    array([ 0,  1,  2,  3, -5])
-
-    """
-    return a.put(ind, v, mode)
-
-
-def swapaxes(a, axis1, axis2):
-    """
-    Interchange two axes of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axis1 : int
-        First axis.
-    axis2 : int
-        Second axis.
-
-    Returns
-    -------
-    a_swapped : ndarray
-        If `a` is an ndarray, then a view of `a` is returned; otherwise
-        a new array is created.
-
-    Examples
-    --------
-    >>> x = np.array([[1,2,3]])
-    >>> np.swapaxes(x,0,1)
-    array([[1],
-           [2],
-           [3]])
-
-    >>> x = np.array([[[0,1],[2,3]],[[4,5],[6,7]]])
-    >>> x
-    array([[[0, 1],
-            [2, 3]],
-           [[4, 5],
-            [6, 7]]])
-
-    >>> np.swapaxes(x,0,2)
-    array([[[0, 4],
-            [2, 6]],
-           [[1, 5],
-            [3, 7]]])
-
-    """
-    try:
-        swapaxes = a.swapaxes
-    except AttributeError:
-        return _wrapit(a, 'swapaxes', axis1, axis2)
-    return swapaxes(axis1, axis2)
-
-
-def transpose(a, axes=None):
-    """
-    Permute the dimensions of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axes : list of ints, optional
-        By default, reverse the dimensions, otherwise permute the axes
-        according to the values given.
-
-    Returns
-    -------
-    p : ndarray
-        `a` with its axes permuted.  A view is returned whenever
-        possible.
-
-    See Also
-    --------
-    rollaxis
-
-    Examples
-    --------
-    >>> x = np.arange(4).reshape((2,2))
-    >>> x
-    array([[0, 1],
-           [2, 3]])
-
-    >>> np.transpose(x)
-    array([[0, 2],
-           [1, 3]])
-
-    >>> x = np.ones((1, 2, 3))
-    >>> np.transpose(x, (1, 0, 2)).shape
-    (2, 1, 3)
-
-    """
-    if axes is not None:
-        raise NotImplementedError('No "axes" arg yet.')
-    try:
-        transpose = a.transpose
-    except AttributeError:
-        return _wrapit(a, 'transpose')
-    return transpose()
-
-
-def partition(a, kth, axis=-1, kind='introselect', order=None):
-    """
-    Return a partitioned copy of an array.
-
-    Creates a copy of the array with its elements rearranged in such a way that
-    the value of the element in kth position is in the position it would be in
-    a sorted array. All elements smaller than the kth element are moved before
-    this element and all equal or greater are moved behind it. The ordering of
-    the elements in the two partitions is undefined.
-
-    .. versionadded:: 1.8.0
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be sorted.
-    kth : int or sequence of ints
-        Element index to partition by. The kth value of the element will be in
-        its final sorted position and all smaller elements will be moved before
-        it and all equal or greater elements behind it.
-        The order all elements in the partitions is undefined.
-        If provided with a sequence of kth it will partition all elements
-        indexed by kth  of them into their sorted position at once.
-    axis : int or None, optional
-        Axis along which to sort. If None, the array is flattened before
-        sorting. The default is -1, which sorts along the last axis.
-    kind : {'introselect'}, optional
-        Selection algorithm. Default is 'introselect'.
-    order : list, optional
-        When `a` is a structured array, this argument specifies which fields
-        to compare first, second, and so on.  This list does not need to
-        include all of the fields.
-
-    Returns
-    -------
-    partitioned_array : ndarray
-        Array of the same type and shape as `a`.
-
-    See Also
-    --------
-    ndarray.partition : Method to sort an array in-place.
-    argpartition : Indirect partition.
-    sort : Full sorting
-
-    Notes
-    -----
-    The various selection algorithms are characterized by their average speed,
-    worst case performance, work space size, and whether they are stable. A
-    stable sort keeps items with the same key in the same relative order. The
-    three available algorithms have the following properties:
-
-    ================= ======= ============= ============ =======
-       kind            speed   worst case    work space  stable
-    ================= ======= ============= ============ =======
-    'introselect'        1        O(n)           0         no
-    ================= ======= ============= ============ =======
-
-    All the partition algorithms make temporary copies of the data when
-    partitioning along any but the last axis.  Consequently, partitioning
-    along the last axis is faster and uses less space than partitioning
-    along any other axis.
-
-    The sort order for complex numbers is lexicographic. If both the real
-    and imaginary parts are non-nan then the order is determined by the
-    real parts except when they are equal, in which case the order is
-    determined by the imaginary parts.
-
-    Examples
-    --------
-    >>> a = np.array([3, 4, 2, 1])
-    >>> np.partition(a, 3)
-    array([2, 1, 3, 4])
-
-    >>> np.partition(a, (1, 3))
-    array([1, 2, 3, 4])
-
-    """
-    if axis is None:
-        a = asanyarray(a).flatten()
-        axis = 0
-    else:
-        a = asanyarray(a).copy()
-    a.partition(kth, axis=axis, kind=kind, order=order)
-    return a
-
-
-def argpartition(a, kth, axis=-1, kind='introselect', order=None):
-    """
-    Perform an indirect partition along the given axis using the algorithm
-    specified by the `kind` keyword. It returns an array of indices of the
-    same shape as `a` that index data along the given axis in partitioned
-    order.
-
-    .. versionadded:: 1.8.0
-
-    Parameters
-    ----------
-    a : array_like
-        Array to sort.
-    kth : int or sequence of ints
-        Element index to partition by. The kth element will be in its final
-        sorted position and all smaller elements will be moved before it and
-        all larger elements behind it.
-        The order all elements in the partitions is undefined.
-        If provided with a sequence of kth it will partition all of them into
-        their sorted position at once.
-    axis : int or None, optional
-        Axis along which to sort.  The default is -1 (the last axis). If None,
-        the flattened array is used.
-    kind : {'introselect'}, optional
-        Selection algorithm. Default is 'introselect'
-    order : list, optional
-        When `a` is an array with fields defined, this argument specifies
-        which fields to compare first, second, etc.  Not all fields need be
-        specified.
-
-    Returns
-    -------
-    index_array : ndarray, int
-        Array of indices that partition `a` along the specified axis.
-        In other words, ``a[index_array]`` yields a sorted `a`.
-
-    See Also
-    --------
-    partition : Describes partition algorithms used.
-    ndarray.partition : Inplace partition.
-    argsort : Full indirect sort
-
-    Notes
-    -----
-    See `partition` for notes on the different selection algorithms.
-
-    Examples
-    --------
-    One dimensional array:
-
-    >>> x = np.array([3, 4, 2, 1])
-    >>> x[np.argpartition(x, 3)]
-    array([2, 1, 3, 4])
-    >>> x[np.argpartition(x, (1, 3))]
-    array([1, 2, 3, 4])
-
-    """
-    return a.argpartition(kth, axis, kind=kind, order=order)
-
-
-def sort(a, axis=-1, kind='quicksort', order=None):
-    """
-    Return a sorted copy of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be sorted.
-    axis : int or None, optional
-        Axis along which to sort. If None, the array is flattened before
-        sorting. The default is -1, which sorts along the last axis.
-    kind : {'quicksort', 'mergesort', 'heapsort'}, optional
-        Sorting algorithm. Default is 'quicksort'.
-    order : list, optional
-        When `a` is a structured array, this argument specifies which fields
-        to compare first, second, and so on.  This list does not need to
-        include all of the fields.
-
-    Returns
-    -------
-    sorted_array : ndarray
-        Array of the same type and shape as `a`.
-
-    See Also
-    --------
-    ndarray.sort : Method to sort an array in-place.
-    argsort : Indirect sort.
-    lexsort : Indirect stable sort on multiple keys.
-    searchsorted : Find elements in a sorted array.
-    partition : Partial sort.
-
-    Notes
-    -----
-    The various sorting algorithms are characterized by their average speed,
-    worst case performance, work space size, and whether they are stable. A
-    stable sort keeps items with the same key in the same relative
-    order. The three available algorithms have the following
-    properties:
-
-    =========== ======= ============= ============ =======
-       kind      speed   worst case    work space  stable
-    =========== ======= ============= ============ =======
-    'quicksort'    1     O(n^2)            0          no
-    'mergesort'    2     O(n*log(n))      ~n/2        yes
-    'heapsort'     3     O(n*log(n))       0          no
-    =========== ======= ============= ============ =======
-
-    All the sort algorithms make temporary copies of the data when
-    sorting along any but the last axis.  Consequently, sorting along
-    the last axis is faster and uses less space than sorting along
-    any other axis.
-
-    The sort order for complex numbers is lexicographic. If both the real
-    and imaginary parts are non-nan then the order is determined by the
-    real parts except when they are equal, in which case the order is
-    determined by the imaginary parts.
-
-    Previous to numpy 1.4.0 sorting real and complex arrays containing nan
-    values led to undefined behaviour. In numpy versions >= 1.4.0 nan
-    values are sorted to the end. The extended sort order is:
-
-      * Real: [R, nan]
-      * Complex: [R + Rj, R + nanj, nan + Rj, nan + nanj]
-
-    where R is a non-nan real value. Complex values with the same nan
-    placements are sorted according to the non-nan part if it exists.
-    Non-nan values are sorted as before.
-
-    Examples
-    --------
-    >>> a = np.array([[1,4],[3,1]])
-    >>> np.sort(a)                # sort along the last axis
-    array([[1, 4],
-           [1, 3]])
-    >>> np.sort(a, axis=None)     # sort the flattened array
-    array([1, 1, 3, 4])
-    >>> np.sort(a, axis=0)        # sort along the first axis
-    array([[1, 1],
-           [3, 4]])
-
-    Use the `order` keyword to specify a field to use when sorting a
-    structured array:
-
-    >>> dtype = [('name', 'S10'), ('height', float), ('age', int)]
-    >>> values = [('Arthur', 1.8, 41), ('Lancelot', 1.9, 38),
-    ...           ('Galahad', 1.7, 38)]
-    >>> a = np.array(values, dtype=dtype)       # create a structured array
-    >>> np.sort(a, order='height')                        # doctest: +SKIP
-    array([('Galahad', 1.7, 38), ('Arthur', 1.8, 41),
-           ('Lancelot', 1.8999999999999999, 38)],
-          dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
-
-    Sort by age, then height if ages are equal:
-
-    >>> np.sort(a, order=['age', 'height'])               # doctest: +SKIP
-    array([('Galahad', 1.7, 38), ('Lancelot', 1.8999999999999999, 38),
-           ('Arthur', 1.8, 41)],
-          dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
-
-    """
-    if axis is None:
-        a = asanyarray(a).flatten()
-        axis = 0
-    else:
-        a = asanyarray(a).copy()
-    a.sort(axis, kind, order)
-    return a
-
-
-def argsort(a, axis=-1, kind='quicksort', order=None):
-    """
-    Returns the indices that would sort an array.
-
-    Perform an indirect sort along the given axis using the algorithm specified
-    by the `kind` keyword. It returns an array of indices of the same shape as
-    `a` that index data along the given axis in sorted order.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to sort.
-    axis : int or None, optional
-        Axis along which to sort.  The default is -1 (the last axis). If None,
-        the flattened array is used.
-    kind : {'quicksort', 'mergesort', 'heapsort'}, optional
-        Sorting algorithm.
-    order : list, optional
-        When `a` is an array with fields defined, this argument specifies
-        which fields to compare first, second, etc.  Not all fields need be
-        specified.
-
-    Returns
-    -------
-    index_array : ndarray, int
-        Array of indices that sort `a` along the specified axis.
-        In other words, ``a[index_array]`` yields a sorted `a`.
-
-    See Also
-    --------
-    sort : Describes sorting algorithms used.
-    lexsort : Indirect stable sort with multiple keys.
-    ndarray.sort : Inplace sort.
-    argpartition : Indirect partial sort.
-
-    Notes
-    -----
-    See `sort` for notes on the different sorting algorithms.
-
-    As of NumPy 1.4.0 `argsort` works with real/complex arrays containing
-    nan values. The enhanced sort order is documented in `sort`.
-
-    Examples
-    --------
-    One dimensional array:
-
-    >>> x = np.array([3, 1, 2])
-    >>> np.argsort(x)
-    array([1, 2, 0])
-
-    Two-dimensional array:
-
-    >>> x = np.array([[0, 3], [2, 2]])
-    >>> x
-    array([[0, 3],
-           [2, 2]])
-
-    >>> np.argsort(x, axis=0)
-    array([[0, 1],
-           [1, 0]])
-
-    >>> np.argsort(x, axis=1)
-    array([[0, 1],
-           [0, 1]])
-
-    Sorting with keys:
-
-    >>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '<i4'), ('y', '<i4')])
-    >>> x
-    array([(1, 0), (0, 1)],
-          dtype=[('x', '<i4'), ('y', '<i4')])
-
-    >>> np.argsort(x, order=('x','y'))
-    array([1, 0])
-
-    >>> np.argsort(x, order=('y','x'))
-    array([0, 1])
-
-    """
-    try:
-        argsort = a.argsort
-    except AttributeError:
-        return _wrapit(a, 'argsort', axis, kind, order)
-    return argsort(axis, kind, order)
-
-
-def argmax(a, axis=None):
-    """
-    Indices of the maximum values along an axis.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axis : int, optional
-        By default, the index is into the flattened array, otherwise
-        along the specified axis.
-
-    Returns
-    -------
-    index_array : ndarray of ints
-        Array of indices into the array. It has the same shape as `a.shape`
-        with the dimension along `axis` removed.
-
-    See Also
-    --------
-    ndarray.argmax, argmin
-    amax : The maximum value along a given axis.
-    unravel_index : Convert a flat index into an index tuple.
-
-    Notes
-    -----
-    In case of multiple occurrences of the maximum values, the indices
-    corresponding to the first occurrence are returned.
-
-    Examples
-    --------
-    >>> a = np.arange(6).reshape(2,3)
-    >>> a
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.argmax(a)
-    5
-    >>> np.argmax(a, axis=0)
-    array([1, 1, 1])
-    >>> np.argmax(a, axis=1)
-    array([2, 2])
-
-    >>> b = np.arange(6)
-    >>> b[1] = 5
-    >>> b
-    array([0, 5, 2, 3, 4, 5])
-    >>> np.argmax(b) # Only the first occurrence is returned.
-    1
-
-    """
-    assert axis is None
-    try:
-        argmax = a.argmax
-    except AttributeError:
-        return _wrapit(a, 'argmax')
-    return argmax()
-
-
-def argmin(a, axis=None):
-    """
-    Return the indices of the minimum values along an axis.
-
-    See Also
-    --------
-    argmax : Similar function.  Please refer to `numpy.argmax` for detailed
-        documentation.
-
-    """
-    assert axis is None
-    try:
-        argmin = a.argmin
-    except AttributeError:
-        return _wrapit(a, 'argmin')
-    return argmin()
-
-
-def searchsorted(a, v, side='left', sorter=None):
-    """
-    Find indices where elements should be inserted to maintain order.
-
-    Find the indices into a sorted array `a` such that, if the
-    corresponding elements in `v` were inserted before the indices, the
-    order of `a` would be preserved.
-
-    Parameters
-    ----------
-    a : 1-D array_like
-        Input array. If `sorter` is None, then it must be sorted in
-        ascending order, otherwise `sorter` must be an array of indices
-        that sort it.
-    v : array_like
-        Values to insert into `a`.
-    side : {'left', 'right'}, optional
-        If 'left', the index of the first suitable location found is given.
-        If 'right', return the last such index.  If there is no suitable
-        index, return either 0 or N (where N is the length of `a`).
-    sorter : 1-D array_like, optional
-        .. versionadded:: 1.7.0
-        Optional array of integer indices that sort array a into ascending
-        order. They are typically the result of argsort.
-
-    Returns
-    -------
-    indices : array of ints
-        Array of insertion points with the same shape as `v`.
-
-    See Also
-    --------
-    sort : Return a sorted copy of an array.
-    histogram : Produce histogram from 1-D data.
-
-    Notes
-    -----
-    Binary search is used to find the required insertion points.
-
-    As of Numpy 1.4.0 `searchsorted` works with real/complex arrays containing
-    `nan` values. The enhanced sort order is documented in `sort`.
-
-    Examples
-    --------
-    >>> np.searchsorted([1,2,3,4,5], 3)
-    2
-    >>> np.searchsorted([1,2,3,4,5], 3, side='right')
-    3
-    >>> np.searchsorted([1,2,3,4,5], [-10, 10, 2, 3])
-    array([0, 5, 1, 2])
-
-    """
-    try:
-        searchsorted = a.searchsorted
-    except AttributeError:
-        return _wrapit(a, 'searchsorted', v, side, sorter)
-    return searchsorted(v, side, sorter)
-
-
-def resize(a, new_shape):
-    """
-    Return a new array with the specified shape.
-
-    If the new array is larger than the original array, then the new
-    array is filled with repeated copies of `a`.  Note that this behavior
-    is different from a.resize(new_shape) which fills with zeros instead
-    of repeated copies of `a`.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be resized.
-
-    new_shape : int or tuple of int
-        Shape of resized array.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        The new array is formed from the data in the old array, repeated
-        if necessary to fill out the required number of elements.  The
-        data are repeated in the order that they are stored in memory.
-
-    See Also
-    --------
-    ndarray.resize : resize an array in-place.
-
-    Examples
-    --------
-    >>> a=np.array([[0,1],[2,3]])
-    >>> np.resize(a,(1,4))
-    array([[0, 1, 2, 3]])
-    >>> np.resize(a,(2,4))
-    array([[0, 1, 2, 3],
-           [0, 1, 2, 3]])
-
-    """
-    if isinstance(new_shape, (int, nt.integer)):
-        new_shape = (new_shape,)
-    a = ravel(a)
-    Na = len(a)
-    if not Na: return mu.zeros(new_shape, a.dtype.char)
-    total_size = um.multiply.reduce(new_shape)
-    n_copies = int(total_size / Na)
-    extra = total_size % Na
-
-    if total_size == 0:

From noreply at buildbot.pypy.org  Wed Oct 30 03:33:58 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 03:33:58 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: change test_lib_pypy/numpypy to
	run against numpy
Message-ID: <20131030023358.8EBAA1C0225@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67724:cd4f6cf1f96e
Date: 2013-10-29 22:15 -0400
http://bitbucket.org/pypy/pypy/changeset/cd4f6cf1f96e/

Log:	change test_lib_pypy/numpypy to run against numpy

diff --git a/pypy/module/test_lib_pypy/numpypy/__init__.py b/pypy/module/test_lib_pypy/numpypy/__init__.py
new file mode 100644
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -1,4 +1,4 @@
-from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
 
 
 class AppTestFromNumeric(BaseNumpyAppTest):
@@ -187,7 +187,9 @@
         x = arange(4).reshape((2,2))
         assert (transpose(x) == array([[0, 2],[1, 3]])).all()
         # Once axes argument is implemented, add more tests
-        raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
 
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
@@ -1,5 +1,4 @@
-
-from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
 
 
 class AppTestBaseRepr(BaseNumpyAppTest):
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_shape_base.py b/pypy/module/test_lib_pypy/numpypy/core/test_shape_base.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_shape_base.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_shape_base.py
@@ -1,4 +1,4 @@
-from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
 
 
 class AppTestShapeBase(BaseNumpyAppTest):
diff --git a/pypy/module/test_lib_pypy/numpypy/lib/test_function_base.py b/pypy/module/test_lib_pypy/numpypy/lib/test_function_base.py
--- a/pypy/module/test_lib_pypy/numpypy/lib/test_function_base.py
+++ b/pypy/module/test_lib_pypy/numpypy/lib/test_function_base.py
@@ -1,4 +1,4 @@
-from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
 
 class AppTestFunctionBase(BaseNumpyAppTest):
     def test_average(self):
diff --git a/pypy/module/test_lib_pypy/numpypy/lib/test_shape_base_lib.py b/pypy/module/test_lib_pypy/numpypy/lib/test_shape_base_lib.py
--- a/pypy/module/test_lib_pypy/numpypy/lib/test_shape_base_lib.py
+++ b/pypy/module/test_lib_pypy/numpypy/lib/test_shape_base_lib.py
@@ -1,4 +1,4 @@
-from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
 
 class AppTestShapeBase(BaseNumpyAppTest):
     def test_dstack(self):
diff --git a/pypy/module/test_lib_pypy/numpypy/lib/test_twodim_base.py b/pypy/module/test_lib_pypy/numpypy/lib/test_twodim_base.py
--- a/pypy/module/test_lib_pypy/numpypy/lib/test_twodim_base.py
+++ b/pypy/module/test_lib_pypy/numpypy/lib/test_twodim_base.py
@@ -1,4 +1,4 @@
-from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
 
 class AppTestTwoDimBase(BaseNumpyAppTest):
     def test_eye(self):
diff --git a/pypy/module/test_lib_pypy/numpypy/test_base.py b/pypy/module/test_lib_pypy/numpypy/test_base.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/numpypy/test_base.py
@@ -0,0 +1,9 @@
+class BaseNumpyAppTest(object):
+    @classmethod
+    def setup_class(cls):
+        if cls.runappdirect:
+            import numpy
+            import sys
+            sys.modules['numpypy'] = numpy
+        else:
+            skip("app-level tests")
diff --git a/pypy/module/test_lib_pypy/numpypy/test_numpy.py b/pypy/module/test_lib_pypy/numpypy/test_numpy.py
--- a/pypy/module/test_lib_pypy/numpypy/test_numpy.py
+++ b/pypy/module/test_lib_pypy/numpypy/test_numpy.py
@@ -1,6 +1,6 @@
 from pypy.conftest import option
 import py, sys
-from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
 
 class AppTestNumpy(BaseNumpyAppTest):
     def test_min_max_after_import(self):

From noreply at buildbot.pypy.org  Wed Oct 30 05:34:13 2013
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 30 Oct 2013 05:34:13 +0100 (CET)
Subject: [pypy-commit] pypy fix-trace-jit: remove now unnecessary return
	values
Message-ID: <20131030043413.539C21C0335@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: fix-trace-jit
Changeset: r67725:ec3bde2a6d36
Date: 2013-10-29 21:33 -0700
http://bitbucket.org/pypy/pypy/changeset/ec3bde2a6d36/

Log:	remove now unnecessary return values

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -272,7 +272,7 @@
 
     def _trace(self, frame, event, w_arg, operr=None):
         if self.is_tracing or frame.hide():
-            return True
+            return
 
         space = self.space
 
@@ -312,7 +312,7 @@
                     event == 'c_call' or
                     event == 'c_return' or
                     event == 'c_exception'):
-                return False
+                return
 
             last_exception = frame.last_exception
             if event == 'leaveframe':
@@ -332,7 +332,6 @@
             finally:
                 frame.last_exception = last_exception
                 self.is_tracing -= 1
-        return False
 
     def checksignals(self):
         """Similar to PyErr_CheckSignals().  If called in the main thread,

From noreply at buildbot.pypy.org  Wed Oct 30 05:54:52 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 05:54:52 +0100 (CET)
Subject: [pypy-commit] pypy default: allow transpose(None)
Message-ID: <20131030045452.582A11C1050@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67726:1606b5779bce
Date: 2013-10-30 00:53 -0400
http://bitbucket.org/pypy/pypy/changeset/1606b5779bce/

Log:	allow transpose(None)

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -345,7 +345,8 @@
         return W_NDimArray(self.implementation.transpose(self))
 
     def descr_transpose(self, space, args_w):
-        if len(args_w) != 0:
+        if not (len(args_w) == 0 or
+                len(args_w) == 1 and space.is_none(args_w[0])):
             raise OperationError(space.w_NotImplementedError, space.wrap(
                 "axes unsupported for transpose"))
         return self.descr_get_transpose(space)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2218,6 +2218,7 @@
         b = a.T
         assert(b[:, 0] == a[0, :]).all()
         assert (a.transpose() == b).all()
+        assert (a.transpose(None) == b).all()
         import sys
         if '__pypy__' in sys.builtin_module_names:
             raises(NotImplementedError, a.transpose, (1, 0, 2))

From noreply at buildbot.pypy.org  Wed Oct 30 07:57:13 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 07:57:13 +0100 (CET)
Subject: [pypy-commit] pypy default: give longfloat its own box so we can
	differentiate
Message-ID: <20131030065713.187EF1C140B@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67727:f14db743ff68
Date: 2013-10-30 02:48 -0400
http://bitbucket.org/pypy/pypy/changeset/f14db743ff68/

Log:	give longfloat its own box so we can differentiate

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -14,6 +14,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rstring import StringBuilder
+from pypy.module.micronumpy.constants import *
 
 
 MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
@@ -352,16 +353,12 @@
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex128")
     _COMPONENTS_BOX = W_Float64Box
 
-if long_double_size == 8:
-    W_FloatLongBox = W_Float64Box
-    W_ComplexLongBox = W_Complex128Box
-
-elif long_double_size in (12, 16):
+if long_double_size in (8, 12, 16):
     class W_FloatLongBox(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % (long_double_size * 8))
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY_LONGDOUBLELTR)
 
     class W_ComplexLongBox(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % (long_double_size * 16))
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY_CLONGDOUBLELTR)
         _COMPONENTS_BOX = W_FloatLongBox
 
 class W_FlexibleBox(W_GenericBox):
@@ -651,7 +648,7 @@
     imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
 )
 
-if long_double_size in (12, 16):
+if long_double_size in (8, 12, 16):
     W_FloatLongBox.typedef = TypeDef("float%d" % (long_double_size * 8), (W_FloatingBox.typedef),
         __module__ = "numpypy",
         __new__ = interp2app(W_FloatLongBox.descr__new__.im_func),
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -642,6 +642,11 @@
             assert numpy.intp is numpy.int64
             assert numpy.uintp is numpy.uint64
 
+        assert issubclass(numpy.float64, numpy.floating)
+        assert issubclass(numpy.longfloat, numpy.floating)
+        assert not issubclass(numpy.float64, numpy.longfloat)
+        assert not issubclass(numpy.longfloat, numpy.float64)
+
     def test_mro(self):
         import numpypy as numpy
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1562,8 +1562,15 @@
     ComponentBoxType = interp_boxes.W_Float64Box
 
 if interp_boxes.long_double_size == 8:
-    FloatLong = Float64
-    ComplexLong = Complex128
+    class FloatLong(BaseType, Float):
+        T = rffi.DOUBLE
+        BoxType = interp_boxes.W_FloatLongBox
+        format_code = "d"
+
+    class ComplexLong(ComplexFloating, BaseType):
+        T = rffi.DOUBLE
+        BoxType = interp_boxes.W_ComplexLongBox
+        ComponentBoxType = interp_boxes.W_Float64Box
 
 elif interp_boxes.long_double_size in (12, 16):
     class FloatLong(BaseType, Float):

From noreply at buildbot.pypy.org  Wed Oct 30 07:57:14 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 07:57:14 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: merge default
Message-ID: <20131030065714.68DCD1C140E@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67728:b2fb28281ff1
Date: 2013-10-30 02:56 -0400
http://bitbucket.org/pypy/pypy/changeset/b2fb28281ff1/

Log:	merge default

diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -14,6 +14,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rstring import StringBuilder
+from pypy.module.micronumpy.constants import *
 
 
 MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
@@ -352,16 +353,12 @@
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex128")
     _COMPONENTS_BOX = W_Float64Box
 
-if long_double_size == 8:
-    W_FloatLongBox = W_Float64Box
-    W_ComplexLongBox = W_Complex128Box
-
-elif long_double_size in (12, 16):
+if long_double_size in (8, 12, 16):
     class W_FloatLongBox(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % (long_double_size * 8))
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY_LONGDOUBLELTR)
 
     class W_ComplexLongBox(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % (long_double_size * 16))
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY_CLONGDOUBLELTR)
         _COMPONENTS_BOX = W_FloatLongBox
 
 class W_FlexibleBox(W_GenericBox):
@@ -651,7 +648,7 @@
     imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
 )
 
-if long_double_size in (12, 16):
+if long_double_size in (8, 12, 16):
     W_FloatLongBox.typedef = TypeDef("float%d" % (long_double_size * 8), (W_FloatingBox.typedef),
         __module__ = "numpypy",
         __new__ = interp2app(W_FloatLongBox.descr__new__.im_func),
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -345,7 +345,8 @@
         return W_NDimArray(self.implementation.transpose(self))
 
     def descr_transpose(self, space, args_w):
-        if len(args_w) != 0:
+        if not (len(args_w) == 0 or
+                len(args_w) == 1 and space.is_none(args_w[0])):
             raise OperationError(space.w_NotImplementedError, space.wrap(
                 "axes unsupported for transpose"))
         return self.descr_get_transpose(space)
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -642,6 +642,11 @@
             assert numpy.intp is numpy.int64
             assert numpy.uintp is numpy.uint64
 
+        assert issubclass(numpy.float64, numpy.floating)
+        assert issubclass(numpy.longfloat, numpy.floating)
+        assert not issubclass(numpy.float64, numpy.longfloat)
+        assert not issubclass(numpy.longfloat, numpy.float64)
+
     def test_mro(self):
         import numpypy as numpy
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2169,6 +2169,7 @@
         b = a.T
         assert(b[:, 0] == a[0, :]).all()
         assert (a.transpose() == b).all()
+        assert (a.transpose(None) == b).all()
         import sys
         if '__pypy__' in sys.builtin_module_names:
             raises(NotImplementedError, a.transpose, (1, 0, 2))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1562,8 +1562,15 @@
     ComponentBoxType = interp_boxes.W_Float64Box
 
 if interp_boxes.long_double_size == 8:
-    FloatLong = Float64
-    ComplexLong = Complex128
+    class FloatLong(BaseType, Float):
+        T = rffi.DOUBLE
+        BoxType = interp_boxes.W_FloatLongBox
+        format_code = "d"
+
+    class ComplexLong(ComplexFloating, BaseType):
+        T = rffi.DOUBLE
+        BoxType = interp_boxes.W_ComplexLongBox
+        ComponentBoxType = interp_boxes.W_Float64Box
 
 elif interp_boxes.long_double_size in (12, 16):
     class FloatLong(BaseType, Float):

From noreply at buildbot.pypy.org  Wed Oct 30 09:37:07 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 09:37:07 +0100 (CET)
Subject: [pypy-commit] pypy default: reshape accepts order (if it's C)
Message-ID: <20131030083707.244E51C1417@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67729:0a32061de4d6
Date: 2013-10-30 04:36 -0400
http://bitbucket.org/pypy/pypy/changeset/0a32061de4d6/

Log:	reshape accepts order (if it's C)

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -311,22 +311,7 @@
                     space.wrap('array does not have imaginary part to set'))
         self.implementation.set_imag(space, self, w_value)
 
-    def descr_reshape(self, space, args_w):
-        """reshape(...)
-        a.reshape(shape)
-
-        Returns an array containing the same data with a new shape.
-
-        Refer to `numpypy.reshape` for full documentation.
-
-        See Also
-        --------
-        numpypy.reshape : equivalent function
-        """
-        if len(args_w) == 1:
-            w_shape = args_w[0]
-        else:
-            w_shape = space.newtuple(args_w)
+    def reshape(self, space, w_shape):
         new_shape = get_shape_from_iterable(space, self.get_size(), w_shape)
         new_impl = self.implementation.reshape(space, self, new_shape)
         if new_impl is not None:
@@ -341,6 +326,36 @@
             arr.implementation.shape = new_shape
         return arr
 
+    def descr_reshape(self, space, __args__):
+        """reshape(...)
+        a.reshape(shape)
+
+        Returns an array containing the same data with a new shape.
+
+        Refer to `numpypy.reshape` for full documentation.
+
+        See Also
+        --------
+        numpypy.reshape : equivalent function
+        """
+        args_w, kw_w = __args__.unpack()
+        order = 'C'
+        if kw_w:
+            if "order" in kw_w:
+                order = space.str_w(kw_w["order"])
+                del kw_w["order"]
+            if kw_w:
+                raise OperationError(space.w_TypeError, space.wrap(
+                    "reshape() got unexpected keyword argument(s)"))
+        if order != 'C':
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "order not implemented"))
+        if len(args_w) == 1:
+            w_shape = args_w[0]
+        else:
+            w_shape = space.newtuple(args_w)
+        return self.reshape(space, w_shape)
+
     def descr_get_transpose(self, space):
         return W_NDimArray(self.implementation.transpose(self))
 
@@ -388,7 +403,7 @@
         if order != 'C':
             raise OperationError(space.w_NotImplementedError, space.wrap(
                 "order not implemented"))
-        return self.descr_reshape(space, [space.wrap(-1)])
+        return self.reshape(space, space.wrap(-1))
 
     @unwrap_spec(w_axis=WrappedDefault(None),
                  w_out=WrappedDefault(None),
@@ -402,14 +417,14 @@
                                  space.wrap("axis unsupported for compress"))
             arr = self
         else:
-            arr = self.descr_reshape(space, [space.wrap(-1)])
+            arr = self.reshape(space, space.wrap(-1))
         index = convert_to_array(space, w_obj)
         return arr.getitem_filter(space, index)
 
     def descr_flatten(self, space, w_order=None):
         if self.is_scalar():
             # scalars have no storage
-            return self.descr_reshape(space, [space.wrap(1)])
+            return self.reshape(space, space.wrap(1))
         w_res = self.descr_ravel(space, w_order)
         if w_res.implementation.storage == self.implementation.storage:
             return w_res.descr_copy(space)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -704,6 +704,11 @@
         (a + a).reshape(2, 12) # assert did not explode
         a = array([[[[]]]])
         assert a.reshape((0,)).shape == (0,)
+        assert a.reshape((0,), order='C').shape == (0,)
+        raises(TypeError, a.reshape, (0,), badarg="C")
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.reshape, (0,), order='F')
 
     def test_slice_reshape(self):
         from numpypy import zeros, arange

From noreply at buildbot.pypy.org  Wed Oct 30 09:49:29 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 09:49:29 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: merge default
Message-ID: <20131030084929.C698A1C0330@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67730:1cffc3a330db
Date: 2013-10-30 04:48 -0400
http://bitbucket.org/pypy/pypy/changeset/1cffc3a330db/

Log:	merge default

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -311,22 +311,7 @@
                     space.wrap('array does not have imaginary part to set'))
         self.implementation.set_imag(space, self, w_value)
 
-    def descr_reshape(self, space, args_w):
-        """reshape(...)
-        a.reshape(shape)
-
-        Returns an array containing the same data with a new shape.
-
-        Refer to `numpypy.reshape` for full documentation.
-
-        See Also
-        --------
-        numpypy.reshape : equivalent function
-        """
-        if len(args_w) == 1:
-            w_shape = args_w[0]
-        else:
-            w_shape = space.newtuple(args_w)
+    def reshape(self, space, w_shape):
         new_shape = get_shape_from_iterable(space, self.get_size(), w_shape)
         new_impl = self.implementation.reshape(space, self, new_shape)
         if new_impl is not None:
@@ -341,6 +326,36 @@
             arr.implementation.shape = new_shape
         return arr
 
+    def descr_reshape(self, space, __args__):
+        """reshape(...)
+        a.reshape(shape)
+
+        Returns an array containing the same data with a new shape.
+
+        Refer to `numpypy.reshape` for full documentation.
+
+        See Also
+        --------
+        numpypy.reshape : equivalent function
+        """
+        args_w, kw_w = __args__.unpack()
+        order = 'C'
+        if kw_w:
+            if "order" in kw_w:
+                order = space.str_w(kw_w["order"])
+                del kw_w["order"]
+            if kw_w:
+                raise OperationError(space.w_TypeError, space.wrap(
+                    "reshape() got unexpected keyword argument(s)"))
+        if order != 'C':
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "order not implemented"))
+        if len(args_w) == 1:
+            w_shape = args_w[0]
+        else:
+            w_shape = space.newtuple(args_w)
+        return self.reshape(space, w_shape)
+
     def descr_get_transpose(self, space):
         return W_NDimArray(self.implementation.transpose(self))
 
@@ -388,7 +403,7 @@
         if order != 'C':
             raise OperationError(space.w_NotImplementedError, space.wrap(
                 "order not implemented"))
-        return self.descr_reshape(space, [space.wrap(-1)])
+        return self.reshape(space, space.wrap(-1))
 
     @unwrap_spec(w_axis=WrappedDefault(None),
                  w_out=WrappedDefault(None),
@@ -402,14 +417,14 @@
                                  space.wrap("axis unsupported for compress"))
             arr = self
         else:
-            arr = self.descr_reshape(space, [space.wrap(-1)])
+            arr = self.reshape(space, space.wrap(-1))
         index = convert_to_array(space, w_obj)
         return arr.getitem_filter(space, index)
 
     def descr_flatten(self, space, w_order=None):
         if self.is_scalar():
             # scalars have no storage
-            return self.descr_reshape(space, [space.wrap(1)])
+            return self.reshape(space, space.wrap(1))
         w_res = self.descr_ravel(space, w_order)
         if w_res.implementation.storage == self.implementation.storage:
             return w_res.descr_copy(space)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -698,6 +698,11 @@
         (a + a).reshape(2, 12) # assert did not explode
         a = array([[[[]]]])
         assert a.reshape((0,)).shape == (0,)
+        assert a.reshape((0,), order='C').shape == (0,)
+        raises(TypeError, a.reshape, (0,), badarg="C")
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.reshape, (0,), order='F')
 
     def test_slice_reshape(self):
         from numpypy import zeros, arange

From noreply at buildbot.pypy.org  Wed Oct 30 09:49:31 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 09:49:31 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: allow test_lib_pypy/numpypy to
 skip if no numpy is installed
Message-ID: <20131030084931.9F69D1C0330@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67731:97ef15b330f1
Date: 2013-10-30 04:47 -0400
http://bitbucket.org/pypy/pypy/changeset/97ef15b330f1/

Log:	allow test_lib_pypy/numpypy to skip if no numpy is installed

diff --git a/pypy/module/test_lib_pypy/numpypy/test_base.py b/pypy/module/test_lib_pypy/numpypy/test_base.py
--- a/pypy/module/test_lib_pypy/numpypy/test_base.py
+++ b/pypy/module/test_lib_pypy/numpypy/test_base.py
@@ -2,7 +2,10 @@
     @classmethod
     def setup_class(cls):
         if cls.runappdirect:
-            import numpy
+            try:
+                import numpy
+            except ImportError:
+                skip("no numpy found")
             import sys
             sys.modules['numpypy'] = numpy
         else:

From noreply at buildbot.pypy.org  Wed Oct 30 10:53:15 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 30 Oct 2013 10:53:15 +0100 (CET)
Subject: [pypy-commit] pypy default: ups fix
Message-ID: <20131030095315.CFB8B1C067F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67732:287e43d11ca8
Date: 2013-10-29 12:35 +0200
http://bitbucket.org/pypy/pypy/changeset/287e43d11ca8/

Log:	ups fix

diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -735,14 +735,15 @@
     r_dict = hop.r_result
     cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
     v_result = hop.gendirectcall(ll_newdict, cDICT)
-    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
-    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
-    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
-        hop.genop('setfield', [v_result, cname, v_eqfn])
-    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
-        hop.genop('setfield', [v_result, cname, v_hashfn])
+    if hasattr(r_dict, 'r_dict_eqfn'):
+        v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
+        v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
+        if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
+            cname = hop.inputconst(lltype.Void, 'fnkeyeq')
+            hop.genop('setfield', [v_result, cname, v_eqfn])
+        if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
+            cname = hop.inputconst(lltype.Void, 'fnkeyhash')
+            hop.genop('setfield', [v_result, cname, v_hashfn])
     return v_result
 
 BUILTIN_TYPER[objectmodel.instantiate] = rtype_instantiate

From noreply at buildbot.pypy.org  Wed Oct 30 10:53:17 2013
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 30 Oct 2013 10:53:17 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20131030095317.833761C1066@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r67733:c6f575f518ae
Date: 2013-10-30 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/c6f575f518ae/

Log:	merge

diff too long, truncating to 2000 out of 2409 lines

diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -121,7 +121,7 @@
         for i, w_index in enumerate(view_w):
             if space.isinstance_w(w_index, space.w_slice):
                 raise IndexError
-            idx = support.int_w(space, w_index)
+            idx = support.index_w(space, w_index)
             if idx < 0:
                 idx = self.get_shape()[i] + idx
             if idx < 0 or idx >= self.get_shape()[i]:
@@ -193,7 +193,7 @@
             return self._lookup_by_index(space, view_w)
         if shape_len > 1:
             raise IndexError
-        idx = support.int_w(space, w_idx)
+        idx = support.index_w(space, w_idx)
         return self._lookup_by_index(space, [space.wrap(idx)])
 
     @jit.unroll_safe
@@ -203,7 +203,7 @@
             dtype = self.dtype
             if not dtype.is_record_type() or idx not in dtype.fields:
                 raise OperationError(space.w_ValueError, space.wrap(
-                    "field named %s not defined" % idx))
+                    "field named %s not found" % idx))
             return RecordChunk(idx)
         if (space.isinstance_w(w_idx, space.w_int) or
             space.isinstance_w(w_idx, space.w_slice)):
diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py
--- a/pypy/module/micronumpy/arrayimpl/scalar.py
+++ b/pypy/module/micronumpy/arrayimpl/scalar.py
@@ -1,4 +1,3 @@
-
 from pypy.module.micronumpy.arrayimpl import base
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy import support
@@ -19,6 +18,9 @@
     def getitem(self):
         return self.v.get_scalar_value()
 
+    def getitem_bool(self):
+        return self.v.dtype.itemtype.bool(self.v.value)
+
     def setitem(self, v):
         self.v.set_scalar_value(v)
 
@@ -181,4 +183,3 @@
     def get_buffer(self, space):
         raise OperationError(space.w_ValueError, space.wrap(
             "cannot point buffer to a scalar"))
-
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -323,8 +323,7 @@
 
 all_types = (types.all_float_types + types.all_complex_types +
              types.all_int_types)
-all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__ and
-                                     not issubclass(i[0], types.BaseFloat16)]
+all_types = [i for i in all_types if not issubclass(i[0], types.Float16)]
 all_types = unrolling_iterable(all_types)
 
 class ArgSortCache(object):
diff --git a/pypy/module/micronumpy/constants.py b/pypy/module/micronumpy/constants.py
--- a/pypy/module/micronumpy/constants.py
+++ b/pypy/module/micronumpy/constants.py
@@ -1,21 +1,83 @@
-from pypy.interpreter.error import OperationError
+NPY_BOOL = 0
+NPY_BYTE = 1
+NPY_UBYTE = 2
+NPY_SHORT = 3
+NPY_USHORT = 4
+NPY_INT = 5
+NPY_UINT = 6
+NPY_LONG = 7
+NPY_ULONG = 8
+NPY_LONGLONG = 9
+NPY_ULONGLONG = 10
+NPY_FLOAT = 11
+NPY_DOUBLE = 12
+NPY_LONGDOUBLE = 13
+NPY_CFLOAT = 14
+NPY_CDOUBLE = 15
+NPY_CLONGDOUBLE = 16
+NPY_OBJECT = 17
+NPY_STRING = 18
+NPY_UNICODE = 19
+NPY_VOID = 20
+NPY_DATETIME = 21
+NPY_TIMEDELTA = 22
+NPY_HALF = 23
+NPY_NTYPES = 24
+NPY_NOTYPE = 25
+NPY_CHAR = 26
+NPY_USERDEF = 256
 
-MODE_CLIP, MODE_WRAP, MODE_RAISE = range(3)
+NPY_BOOLLTR = '?'
+NPY_BYTELTR = 'b'
+NPY_UBYTELTR = 'B'
+NPY_SHORTLTR = 'h'
+NPY_USHORTLTR = 'H'
+NPY_INTLTR = 'i'
+NPY_UINTLTR = 'I'
+NPY_LONGLTR = 'l'
+NPY_ULONGLTR = 'L'
+NPY_LONGLONGLTR = 'q'
+NPY_ULONGLONGLTR = 'Q'
+NPY_HALFLTR = 'e'
+NPY_FLOATLTR = 'f'
+NPY_DOUBLELTR = 'd'
+NPY_LONGDOUBLELTR = 'g'
+NPY_CFLOATLTR = 'F'
+NPY_CDOUBLELTR = 'D'
+NPY_CLONGDOUBLELTR = 'G'
+NPY_OBJECTLTR = 'O'
+NPY_STRINGLTR = 'S'
+NPY_STRINGLTR2 = 'a'
+NPY_UNICODELTR = 'U'
+NPY_VOIDLTR = 'V'
+NPY_DATETIMELTR = 'M'
+NPY_TIMEDELTALTR = 'm'
+NPY_CHARLTR = 'c'
 
-def clipmode_converter(space, w_mode):
-    if space.is_none(w_mode):
-        return MODE_RAISE
-    if space.isinstance_w(w_mode, space.w_str):
-        mode = space.str_w(w_mode)
-        if mode.startswith('C') or mode.startswith('c'):
-            return MODE_CLIP
-        if mode.startswith('W') or mode.startswith('w'):
-            return MODE_WRAP
-        if mode.startswith('R') or mode.startswith('r'):
-            return MODE_RAISE
-    elif space.isinstance_w(w_mode, space.w_int):
-        mode = space.int_w(w_mode)
-        if MODE_CLIP <= mode <= MODE_RAISE:
-            return mode
-    raise OperationError(space.w_TypeError,
-                         space.wrap("clipmode not understood"))
+NPY_INTPLTR = 'p'
+NPY_UINTPLTR = 'P'
+
+NPY_GENBOOLLTR ='b'
+NPY_SIGNEDLTR = 'i'
+NPY_UNSIGNEDLTR = 'u'
+NPY_FLOATINGLTR = 'f'
+NPY_COMPLEXLTR = 'c'
+
+NPY_CLIP = 0
+NPY_WRAP = 1
+NPY_RAISE = 2
+
+NPY_LITTLE = '<'
+NPY_BIG = '>'
+NPY_NATIVE = '='
+NPY_SWAP = 's'
+NPY_IGNORE = '|'
+
+import sys
+if sys.byteorder == 'big':
+    NPY_NATBYTE = NPY_BIG
+    NPY_OPPBYTE = NPY_LITTLE
+else:
+    NPY_NATBYTE = NPY_LITTLE
+    NPY_OPPBYTE = NPY_BIG
+del sys
diff --git a/pypy/module/micronumpy/conversion_utils.py b/pypy/module/micronumpy/conversion_utils.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/conversion_utils.py
@@ -0,0 +1,20 @@
+from pypy.interpreter.error import OperationError
+from pypy.module.micronumpy.constants import *
+
+def clipmode_converter(space, w_mode):
+    if space.is_none(w_mode):
+        return NPY_RAISE
+    if space.isinstance_w(w_mode, space.w_str):
+        mode = space.str_w(w_mode)
+        if mode.startswith('C') or mode.startswith('c'):
+            return NPY_CLIP
+        if mode.startswith('W') or mode.startswith('w'):
+            return NPY_WRAP
+        if mode.startswith('R') or mode.startswith('r'):
+            return NPY_RAISE
+    elif space.isinstance_w(w_mode, space.w_int):
+        mode = space.int_w(w_mode)
+        if NPY_CLIP <= mode <= NPY_RAISE:
+            return mode
+    raise OperationError(space.w_TypeError,
+                         space.wrap("clipmode not understood"))
diff --git a/pypy/module/micronumpy/dot.py b/pypy/module/micronumpy/dot.py
deleted file mode 100644
--- a/pypy/module/micronumpy/dot.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from pypy.interpreter.error import OperationError
-
-def match_dot_shapes(space, left, right):
-    left_shape = left.get_shape()
-    right_shape = right.get_shape()
-    my_critical_dim_size = left_shape[-1]
-    right_critical_dim_size = right_shape[0]
-    right_critical_dim = 0
-    out_shape = []
-    if len(right_shape) > 1:
-        right_critical_dim = len(right_shape) - 2
-        right_critical_dim_size = right_shape[right_critical_dim]
-        assert right_critical_dim >= 0
-        out_shape = out_shape + left_shape[:-1] + \
-                    right_shape[0:right_critical_dim] + \
-                    right_shape[right_critical_dim + 1:]
-    elif len(right_shape) > 0:
-        #dot does not reduce for scalars
-        out_shape = out_shape + left_shape[:-1]
-    if my_critical_dim_size != right_critical_dim_size:
-        raise OperationError(space.w_ValueError, space.wrap(
-                                        "objects are not aligned"))
-    return out_shape, right_critical_dim
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -1,11 +1,12 @@
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
-from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs, constants
+from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs
 from pypy.module.micronumpy.iter import Chunk, Chunks
 from pypy.module.micronumpy.strides import shape_agreement,\
      shape_agreement_multiple
-from pypy.module.micronumpy.constants import clipmode_converter
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
+from pypy.module.micronumpy.conversion_utils import clipmode_converter
+from pypy.module.micronumpy.constants import *
 
 def where(space, w_arr, w_x=None, w_y=None):
     """where(condition, [x, y])
@@ -190,7 +191,7 @@
     return out
 
 def put(space, w_arr, w_indices, w_values, w_mode):
-    from pypy.module.micronumpy.support import int_w
+    from pypy.module.micronumpy.support import index_w
 
     arr = convert_to_array(space, w_arr)
     mode = clipmode_converter(space, w_mode)
@@ -216,20 +217,21 @@
 
     v_idx = 0
     for idx in indices:
-        index = int_w(space, idx)
+        index = index_w(space, idx)
 
         if index < 0 or index >= arr.get_size():
-            if mode == constants.MODE_RAISE:
+            if mode == NPY_RAISE:
                 raise OperationError(space.w_IndexError, space.wrap(
                     "index %d is out of bounds for axis 0 with size %d" % (index, arr.get_size())))
-            elif mode == constants.MODE_WRAP:
+            elif mode == NPY_WRAP:
                 index = index % arr.get_size()
-            else:
-                assert mode == constants.MODE_CLIP
+            elif mode == NPY_CLIP:
                 if index < 0:
                     index = 0
                 else:
                     index = arr.get_size() - 1
+            else:
+                assert False
 
         value = values[v_idx]
 
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -14,6 +14,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rstring import StringBuilder
+from pypy.module.micronumpy.constants import *
 
 
 MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
@@ -34,7 +35,7 @@
         from pypy.module.micronumpy.interp_dtype import get_dtype_cache
         return get_dtype_cache(space).dtypes_by_name[name]
 
-    def new(space, w_subtype, w_value):
+    def new(space, w_subtype, w_value=None):
         dtype = _get_dtype(space)
         return dtype.itemtype.coerce_subtype(space, w_subtype, w_value)
 
@@ -352,16 +353,12 @@
     descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex128")
     _COMPONENTS_BOX = W_Float64Box
 
-if long_double_size == 8:
-    W_FloatLongBox = W_Float64Box
-    W_ComplexLongBox = W_Complex128Box
-
-elif long_double_size in (12, 16):
+if long_double_size in (8, 12, 16):
     class W_FloatLongBox(W_FloatingBox, PrimitiveBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("float%d" % (long_double_size * 8))
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY_LONGDOUBLELTR)
 
     class W_ComplexLongBox(ComplexBox, W_ComplexFloatingBox):
-        descr__new__, _get_dtype, descr_reduce = new_dtype_getter("complex%d" % (long_double_size * 16))
+        descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY_CLONGDOUBLELTR)
         _COMPONENTS_BOX = W_FloatLongBox
 
 class W_FlexibleBox(W_GenericBox):
@@ -378,25 +375,27 @@
 
 class W_VoidBox(W_FlexibleBox):
     def descr_getitem(self, space, w_item):
-        from pypy.module.micronumpy.types import VoidType
-        if space.isinstance_w(w_item, space.w_str):
+        if space.isinstance_w(w_item, space.w_basestring):
             item = space.str_w(w_item)
         elif space.isinstance_w(w_item, space.w_int):
-            #Called by iterator protocol
             indx = space.int_w(w_item)
             try:
                 item = self.dtype.fieldnames[indx]
             except IndexError:
-                raise OperationError(space.w_IndexError,
-                     space.wrap("Iterated over too many fields %d" % indx))
+                if indx < 0:
+                    indx += len(self.dtype.fieldnames)
+                raise OperationError(space.w_IndexError, space.wrap(
+                    "invalid index (%d)" % indx))
         else:
             raise OperationError(space.w_IndexError, space.wrap(
-                    "Can only access fields of record with int or str"))
+                "invalid index"))
         try:
             ofs, dtype = self.dtype.fields[item]
         except KeyError:
-            raise OperationError(space.w_IndexError,
-                                 space.wrap("Field %s does not exist" % item))
+            raise OperationError(space.w_IndexError, space.wrap(
+                "invalid index"))
+
+        from pypy.module.micronumpy.types import VoidType
         if isinstance(dtype.itemtype, VoidType):
             read_val = dtype.itemtype.readarray(self.arr, self.ofs, ofs, dtype)
         else:
@@ -649,7 +648,7 @@
     imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag),
 )
 
-if long_double_size in (12, 16):
+if long_double_size in (8, 12, 16):
     W_FloatLongBox.typedef = TypeDef("float%d" % (long_double_size * 8), (W_FloatingBox.typedef),
         __module__ = "numpypy",
         __new__ = interp2app(W_FloatLongBox.descr__new__.im_func),
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -9,24 +9,8 @@
 from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib import jit
+from pypy.module.micronumpy.constants import *
 
-if sys.byteorder == 'little':
-    NATBYTE = '<'
-    OPPBYTE = '>'
-else:
-    NATBYTE = '>'
-    OPPBYTE = '<'
-
-UNSIGNEDLTR = "u"
-SIGNEDLTR = "i"
-BOOLLTR = "b"
-FLOATINGLTR = "f"
-COMPLEXLTR = "c"
-VOIDLTR = 'V'
-STRINGLTR = 'S'
-UNICODELTR = 'U'
-INTPLTR = 'p'
-UINTPLTR = 'P'
 
 def decode_w_dtype(space, w_dtype):
     if space.is_none(w_dtype):
@@ -52,7 +36,7 @@
 class W_Dtype(W_Root):
     _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder", "float_type"]
 
-    def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder='=',
+    def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder=NPY_NATIVE,
                  alternate_constructors=[], aliases=[], float_type=None,
                  fields=None, fieldnames=None, shape=[], subdtype=None):
         self.itemtype = itemtype
@@ -107,35 +91,35 @@
         self.itemtype.fill(storage, self.get_size(), box, start, stop, 0)
 
     def is_int_type(self):
-        return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or
-                self.kind == BOOLLTR)
+        return (self.kind == NPY_SIGNEDLTR or self.kind == NPY_UNSIGNEDLTR or
+                self.kind == NPY_GENBOOLLTR)
 
     def is_signed(self):
-        return self.kind == SIGNEDLTR
+        return self.kind == NPY_SIGNEDLTR
 
     def is_complex_type(self):
-        return self.kind == COMPLEXLTR
+        return self.kind == NPY_COMPLEXLTR
 
     def is_float_type(self):
-        return (self.kind == FLOATINGLTR or self.float_type is not None)
+        return (self.kind == NPY_FLOATINGLTR or self.float_type is not None)
 
     def is_bool_type(self):
-        return self.kind == BOOLLTR
+        return self.kind == NPY_GENBOOLLTR
 
     def is_record_type(self):
         return self.fields is not None
 
     def is_str_type(self):
-        return self.num == 18
+        return self.num == NPY_STRING
 
     def is_str_or_unicode(self):
-        return (self.num == 18 or self.num == 19)
+        return (self.num == NPY_STRING or self.num == NPY_UNICODE)
 
     def is_flexible_type(self):
         return (self.is_str_or_unicode() or self.is_record_type())
 
     def is_native(self):
-        return self.byteorder in ('=', NATBYTE)
+        return self.byteorder in (NPY_NATIVE, NPY_NATBYTE)
 
     def get_size(self):
         return self.itemtype.get_element_size()
@@ -163,15 +147,15 @@
     def descr_get_str(self, space):
         size = self.get_size()
         basic = self.kind
-        if basic == UNICODELTR:
+        if basic == NPY_UNICODELTR:
             size >>= 2
-            endian = NATBYTE
+            endian = NPY_NATBYTE
         elif size <= 1:
-            endian = '|'  # ignore
+            endian = NPY_IGNORE
         else:
             endian = self.byteorder
-            if endian == '=':
-                endian = NATBYTE
+            if endian == NPY_NATIVE:
+                endian = NPY_NATBYTE
         return space.wrap("%s%s%s" % (endian, basic, size))
 
     def descr_get_base(self, space):
@@ -248,14 +232,27 @@
                         raise
                     break
 
-    @unwrap_spec(item=str)
-    def descr_getitem(self, space, item):
+    def descr_getitem(self, space, w_item):
         if self.fields is None:
-            raise OperationError(space.w_KeyError, space.wrap("There are no keys in dtypes %s" % self.name))
+            raise OperationError(space.w_KeyError, space.wrap(
+                "There are no fields in dtype %s." % self.name))
+        if space.isinstance_w(w_item, space.w_basestring):
+            item = space.str_w(w_item)
+        elif space.isinstance_w(w_item, space.w_int):
+            indx = space.int_w(w_item)
+            try:
+                item = self.fieldnames[indx]
+            except IndexError:
+                raise OperationError(space.w_IndexError, space.wrap(
+                    "Field index %d out of range." % indx))
+        else:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "Field key must be an integer, string, or unicode."))
         try:
             return self.fields[item][1]
         except KeyError:
-            raise OperationError(space.w_KeyError, space.wrap("Field named %s not found" % item))
+            raise OperationError(space.w_KeyError, space.wrap(
+                "Field named '%s' not found." % item))
 
     def descr_reduce(self, space):
         w_class = space.type(self)
@@ -268,7 +265,7 @@
         names = self.descr_get_names(space)
         values = self.descr_get_fields(space)
         if self.fields:
-            endian = '|'
+            endian = NPY_IGNORE
             #TODO: Implement this when subarrays are implemented
             subdescr = space.w_None
             size = 0
@@ -281,8 +278,8 @@
             alignment = space.wrap(1)
         else:
             endian = self.byteorder
-            if endian == '=':
-                endian = NATBYTE
+            if endian == NPY_NATIVE:
+                endian = NPY_NATBYTE
             subdescr = space.w_None
             w_size = space.wrap(-1)
             alignment = space.wrap(-1)
@@ -296,8 +293,8 @@
             raise OperationError(space.w_NotImplementedError, space.wrap("Pickling protocol version not supported"))
 
         endian = space.str_w(space.getitem(w_data, space.wrap(1)))
-        if endian == NATBYTE:
-            endian = '='
+        if endian == NPY_NATBYTE:
+            endian = NPY_NATIVE
         self.byteorder = endian
 
         fieldnames = space.getitem(w_data, space.wrap(3))
@@ -331,8 +328,8 @@
         offset += subdtype.itemtype.get_element_size() * size
         fieldnames.append(fldname)
     itemtype = types.RecordType(ofs_and_items, offset)
-    return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
-                   "V", space.gettypefor(interp_boxes.W_VoidBox), fields=fields,
+    return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
+                   NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), fields=fields,
                    fieldnames=fieldnames)
 
 def dtype_from_dict(space, w_dict):
@@ -358,8 +355,8 @@
             dim = space.int_w(w_dim)
             shape.append(dim)
             size *= dim
-        return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), 20, VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size),
-                    "V", space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype)
+        return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), NPY_VOID, NPY_VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size),
+                    NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype)
 
     if space.is_none(w_dtype):
         return cache.w_float64dtype
@@ -435,26 +432,28 @@
             size = int(name[1:])
         except ValueError:
             raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
-    if char == 'c':
-        char = 'S'
+    if char == NPY_CHARLTR:
+        char = NPY_STRINGLTR
         size = 1
-    if char == 'S':
+
+    if char == NPY_STRINGLTR:
         itemtype = types.StringType(size)
         basename = 'string'
-        num = 18
+        num = NPY_STRING
         w_box_type = space.gettypefor(interp_boxes.W_StringBox)
-    elif char == 'V':
-        num = 20
+    elif char == NPY_VOIDLTR:
+        itemtype = types.VoidType(size)
         basename = 'void'
-        itemtype = types.VoidType(size)
-        return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(size),
-                    "V", space.gettypefor(interp_boxes.W_VoidBox))
+        num = NPY_VOID
+        w_box_type = space.gettypefor(interp_boxes.W_VoidBox)
+    elif char == NPY_UNICODELTR:
+        itemtype = types.UnicodeType(size)
+        basename = 'unicode'
+        num = NPY_UNICODE
+        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
     else:
-        assert char == 'U'
-        basename = 'unicode'
-        itemtype = types.UnicodeType(size)
-        num = 19
-        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
+        assert False
+
     return W_Dtype(itemtype, num, char,
                    basename + str(8 * itemtype.get_element_size()),
                    char, w_box_type)
@@ -463,10 +462,10 @@
     itemtype = types.StringType(size)
     return W_Dtype(
         itemtype,
-        num=18,
-        kind=STRINGLTR,
+        num=NPY_STRING,
+        kind=NPY_STRINGLTR,
         name='string' + str(8 * itemtype.get_element_size()),
-        char='S',
+        char=NPY_STRINGLTR,
         w_box_type = space.gettypefor(interp_boxes.W_StringBox),
     )
 
@@ -474,10 +473,10 @@
     itemtype = types.UnicodeType(size)
     return W_Dtype(
         itemtype,
-        num=19,
-        kind=UNICODELTR,
+        num=NPY_UNICODE,
+        kind=NPY_UNICODELTR,
         name='unicode' + str(8 * itemtype.get_element_size()),
-        char='U',
+        char=NPY_UNICODELTR,
         w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
     )
 
@@ -486,67 +485,72 @@
     def __init__(self, space):
         self.w_booldtype = W_Dtype(
             types.Bool(),
-            num=0,
-            kind=BOOLLTR,
+            num=NPY_BOOL,
+            kind=NPY_GENBOOLLTR,
             name="bool",
-            char="?",
+            char=NPY_BOOLLTR,
             w_box_type=space.gettypefor(interp_boxes.W_BoolBox),
             alternate_constructors=[space.w_bool],
+            aliases=['bool8'],
         )
         self.w_int8dtype = W_Dtype(
             types.Int8(),
-            num=1,
-            kind=SIGNEDLTR,
+            num=NPY_BYTE,
+            kind=NPY_SIGNEDLTR,
             name="int8",
-            char="b",
-            w_box_type=space.gettypefor(interp_boxes.W_Int8Box)
+            char=NPY_BYTELTR,
+            w_box_type=space.gettypefor(interp_boxes.W_Int8Box),
+            aliases=['byte'],
         )
         self.w_uint8dtype = W_Dtype(
             types.UInt8(),
-            num=2,
-            kind=UNSIGNEDLTR,
+            num=NPY_UBYTE,
+            kind=NPY_UNSIGNEDLTR,
             name="uint8",
-            char="B",
+            char=NPY_UBYTELTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt8Box),
+            aliases=['ubyte'],
         )
         self.w_int16dtype = W_Dtype(
             types.Int16(),
-            num=3,
-            kind=SIGNEDLTR,
+            num=NPY_SHORT,
+            kind=NPY_SIGNEDLTR,
             name="int16",
-            char="h",
+            char=NPY_SHORTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int16Box),
+            aliases=['short'],
         )
         self.w_uint16dtype = W_Dtype(
             types.UInt16(),
-            num=4,
-            kind=UNSIGNEDLTR,
+            num=NPY_USHORT,
+            kind=NPY_UNSIGNEDLTR,
             name="uint16",
-            char="H",
+            char=NPY_USHORTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt16Box),
+            aliases=['ushort'],
         )
         self.w_int32dtype = W_Dtype(
             types.Int32(),
-            num=5,
-            kind=SIGNEDLTR,
+            num=NPY_INT,
+            kind=NPY_SIGNEDLTR,
             name="int32",
-            char="i",
+            char=NPY_INTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int32Box),
        )
         self.w_uint32dtype = W_Dtype(
             types.UInt32(),
-            num=6,
-            kind=UNSIGNEDLTR,
+            num=NPY_UINT,
+            kind=NPY_UNSIGNEDLTR,
             name="uint32",
-            char="I",
+            char=NPY_UINTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt32Box),
         )
         self.w_longdtype = W_Dtype(
             types.Long(),
-            num=7,
-            kind=SIGNEDLTR,
+            num=NPY_LONG,
+            kind=NPY_SIGNEDLTR,
             name="int%d" % LONG_BIT,
-            char="l",
+            char=NPY_LONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_LongBox),
             alternate_constructors=[space.w_int,
                                     space.gettypefor(interp_boxes.W_IntegerBox),
@@ -556,10 +560,10 @@
         )
         self.w_ulongdtype = W_Dtype(
             types.ULong(),
-            num=8,
-            kind=UNSIGNEDLTR,
+            num=NPY_ULONG,
+            kind=NPY_UNSIGNEDLTR,
             name="uint%d" % LONG_BIT,
-            char="L",
+            char=NPY_ULONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
             alternate_constructors=[ space.gettypefor(interp_boxes.W_UnsignedIntegerBox),
                                    ],
@@ -567,35 +571,38 @@
         )
         self.w_int64dtype = W_Dtype(
             types.Int64(),
-            num=9,
-            kind=SIGNEDLTR,
+            num=NPY_LONGLONG,
+            kind=NPY_SIGNEDLTR,
             name="int64",
-            char="q",
+            char=NPY_LONGLONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
             alternate_constructors=[space.w_long],
+            aliases=['longlong'],
         )
         self.w_uint64dtype = W_Dtype(
             types.UInt64(),
-            num=10,
-            kind=UNSIGNEDLTR,
+            num=NPY_ULONGLONG,
+            kind=NPY_UNSIGNEDLTR,
             name="uint64",
-            char="Q",
+            char=NPY_ULONGLONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_UInt64Box),
+            aliases=['ulonglong'],
         )
         self.w_float32dtype = W_Dtype(
             types.Float32(),
-            num=11,
-            kind=FLOATINGLTR,
+            num=NPY_FLOAT,
+            kind=NPY_FLOATINGLTR,
             name="float32",
-            char="f",
+            char=NPY_FLOATLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Float32Box),
+            aliases=['single']
         )
         self.w_float64dtype = W_Dtype(
             types.Float64(),
-            num=12,
-            kind=FLOATINGLTR,
+            num=NPY_DOUBLE,
+            kind=NPY_FLOATINGLTR,
             name="float64",
-            char="d",
+            char=NPY_DOUBLELTR,
             w_box_type = space.gettypefor(interp_boxes.W_Float64Box),
             alternate_constructors=[space.w_float,
                                     space.gettypefor(interp_boxes.W_NumberBox),
@@ -604,68 +611,69 @@
         )
         self.w_floatlongdtype = W_Dtype(
             types.FloatLong(),
-            num=13,
-            kind=FLOATINGLTR,
+            num=NPY_LONGDOUBLE,
+            kind=NPY_FLOATINGLTR,
             name="float%d" % (interp_boxes.long_double_size * 8),
-            char="g",
+            char=NPY_LONGDOUBLELTR,
             w_box_type=space.gettypefor(interp_boxes.W_FloatLongBox),
             aliases=["longdouble", "longfloat"],
         )
         self.w_complex64dtype = W_Dtype(
             types.Complex64(),
-            num=14,
-            kind=COMPLEXLTR,
+            num=NPY_CFLOAT,
+            kind=NPY_COMPLEXLTR,
             name="complex64",
-            char="F",
+            char=NPY_CFLOATLTR,
             w_box_type = space.gettypefor(interp_boxes.W_Complex64Box),
+            aliases=['csingle'],
             float_type = self.w_float32dtype,
         )
         self.w_complex128dtype = W_Dtype(
             types.Complex128(),
-            num=15,
-            kind=COMPLEXLTR,
+            num=NPY_CDOUBLE,
+            kind=NPY_COMPLEXLTR,
             name="complex128",
-            char="D",
+            char=NPY_CDOUBLELTR,
             w_box_type = space.gettypefor(interp_boxes.W_Complex128Box),
             alternate_constructors=[space.w_complex],
-            aliases=["complex"],
+            aliases=["complex", 'cfloat', 'cdouble'],
             float_type = self.w_float64dtype,
         )
         self.w_complexlongdtype = W_Dtype(
             types.ComplexLong(),
-            num=16,
-            kind=COMPLEXLTR,
+            num=NPY_CLONGDOUBLE,
+            kind=NPY_COMPLEXLTR,
             name="complex%d" % (interp_boxes.long_double_size * 16),
-            char="G",
+            char=NPY_CLONGDOUBLELTR,
             w_box_type = space.gettypefor(interp_boxes.W_ComplexLongBox),
             aliases=["clongdouble", "clongfloat"],
             float_type = self.w_floatlongdtype,
         )
         self.w_stringdtype = W_Dtype(
             types.StringType(0),
-            num=18,
-            kind=STRINGLTR,
+            num=NPY_STRING,
+            kind=NPY_STRINGLTR,
             name='string',
-            char='S',
+            char=NPY_STRINGLTR,
             w_box_type = space.gettypefor(interp_boxes.W_StringBox),
             alternate_constructors=[space.w_str, space.gettypefor(interp_boxes.W_CharacterBox)],
             aliases=["str"],
         )
         self.w_unicodedtype = W_Dtype(
             types.UnicodeType(0),
-            num=19,
-            kind=UNICODELTR,
+            num=NPY_UNICODE,
+            kind=NPY_UNICODELTR,
             name='unicode',
-            char='U',
+            char=NPY_UNICODELTR,
             w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
             alternate_constructors=[space.w_unicode],
         )
         self.w_voiddtype = W_Dtype(
             types.VoidType(0),
-            num=20,
-            kind=VOIDLTR,
+            num=NPY_VOID,
+            kind=NPY_VOIDLTR,
             name='void',
-            char='V',
+            char=NPY_VOIDLTR,
             w_box_type = space.gettypefor(interp_boxes.W_VoidBox),
             #alternate_constructors=[space.w_buffer],
             # XXX no buffer in space
@@ -674,43 +682,43 @@
         )
         self.w_float16dtype = W_Dtype(
             types.Float16(),
-            num=23,
-            kind=FLOATINGLTR,
+            num=NPY_HALF,
+            kind=NPY_FLOATINGLTR,
             name="float16",
-            char="e",
+            char=NPY_HALFLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Float16Box),
         )
         ptr_size = rffi.sizeof(rffi.CCHARP)
         if ptr_size == 4:
             intp_box = interp_boxes.W_Int32Box
             intp_type = types.Int32()
-            intp_num = 5
+            intp_num = NPY_INT
             uintp_box = interp_boxes.W_UInt32Box
             uintp_type = types.UInt32()
-            uintp_num = 6
+            uintp_num = NPY_UINT
         elif ptr_size == 8:
             intp_box = interp_boxes.W_Int64Box
             intp_type = types.Int64()
-            intp_num = 7
+            intp_num = NPY_LONG
             uintp_box = interp_boxes.W_UInt64Box
             uintp_type = types.UInt64()
-            uintp_num = 8
+            uintp_num = NPY_ULONG
         else:
             raise ValueError('unknown point size %d' % ptr_size)
         self.w_intpdtype = W_Dtype(
             intp_type,
             num=intp_num,
-            kind=INTPLTR,
+            kind=NPY_INTPLTR,
             name='intp',
-            char=INTPLTR,
+            char=NPY_INTPLTR,
             w_box_type = space.gettypefor(intp_box),
         )
         self.w_uintpdtype = W_Dtype(
             uintp_type,
             num=uintp_num,
-            kind=UINTPLTR,
+            kind=NPY_UINTPLTR,
             name='uintp',
-            char=UINTPLTR,
+            char=NPY_UINTPLTR,
             w_box_type = space.gettypefor(uintp_box),
         )
         float_dtypes = [self.w_float16dtype, self.w_float32dtype,
@@ -741,24 +749,23 @@
             self.dtypes_by_name[dtype.name] = dtype
             can_name = dtype.kind + str(dtype.itemtype.get_element_size())
             self.dtypes_by_name[can_name] = dtype
-            self.dtypes_by_name[NATBYTE + can_name] = dtype
-            self.dtypes_by_name['=' + can_name] = dtype
-            new_name = OPPBYTE + can_name
-            itemtypename = dtype.itemtype.__class__.__name__
-            itemtype = getattr(types, 'NonNative' + itemtypename)()
+            self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype
+            self.dtypes_by_name[NPY_NATIVE + can_name] = dtype
+            new_name = NPY_OPPBYTE + can_name
+            itemtype = type(dtype.itemtype)(False)
             self.dtypes_by_name[new_name] = W_Dtype(
                 itemtype,
                 dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                byteorder=OPPBYTE, float_type=dtype.float_type)
+                byteorder=NPY_OPPBYTE, float_type=dtype.float_type)
             if dtype.kind != dtype.char:
                 can_name = dtype.char
-                self.dtypes_by_name[NATBYTE + can_name] = dtype
-                self.dtypes_by_name['=' + can_name] = dtype
-                new_name = OPPBYTE + can_name
+                self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype
+                self.dtypes_by_name[NPY_NATIVE + can_name] = dtype
+                new_name = NPY_OPPBYTE + can_name
                 self.dtypes_by_name[new_name] = W_Dtype(
                     itemtype,
                     dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                    byteorder=OPPBYTE, float_type=dtype.float_type)
+                    byteorder=NPY_OPPBYTE, float_type=dtype.float_type)
 
             for alias in dtype.aliases:
                 self.dtypes_by_name[alias] = dtype
@@ -815,9 +822,9 @@
                        space.wrap(dtype.num),
                        space.wrap(itemsize * 8), # in case of changing
                        # number of bits per byte in the future
-                       space.wrap(itemsize / (2 if dtype.kind == COMPLEXLTR else 1) or 1)]
+                       space.wrap(itemsize / (2 if dtype.kind == NPY_COMPLEXLTR else 1) or 1)]
             if dtype.is_int_type():
-                if dtype.kind == BOOLLTR:
+                if dtype.kind == NPY_GENBOOLLTR:
                     w_maxobj = space.wrap(1)
                     w_minobj = space.wrap(0)
                 elif dtype.is_signed():
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -12,7 +12,6 @@
 from pypy.module.micronumpy.interp_flatiter import W_FlatIterator
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy import loop
-from pypy.module.micronumpy.dot import match_dot_shapes
 from pypy.module.micronumpy.interp_arrayops import repeat, choose, put
 from pypy.module.micronumpy.arrayimpl import scalar
 from rpython.tool.sourcetools import func_with_new_name
@@ -31,6 +30,28 @@
     shape += dtype.shape
     return shape[:]
 
+def _match_dot_shapes(space, left, right):
+    left_shape = left.get_shape()
+    right_shape = right.get_shape()
+    my_critical_dim_size = left_shape[-1]
+    right_critical_dim_size = right_shape[0]
+    right_critical_dim = 0
+    out_shape = []
+    if len(right_shape) > 1:
+        right_critical_dim = len(right_shape) - 2
+        right_critical_dim_size = right_shape[right_critical_dim]
+        assert right_critical_dim >= 0
+        out_shape = out_shape + left_shape[:-1] + \
+                    right_shape[0:right_critical_dim] + \
+                    right_shape[right_critical_dim + 1:]
+    elif len(right_shape) > 0:
+        #dot does not reduce for scalars
+        out_shape = out_shape + left_shape[:-1]
+    if my_critical_dim_size != right_critical_dim_size:
+        raise OperationError(space.w_ValueError, space.wrap(
+                                        "objects are not aligned"))
+    return out_shape, right_critical_dim
+
 class __extend__(W_NDimArray):
     @jit.unroll_safe
     def descr_get_shape(self, space):
@@ -180,8 +201,6 @@
             return self.implementation.descr_getitem(space, self, w_idx)
         except ArrayArgumentException:
             return self.getitem_array_int(space, w_idx)
-        except OperationError:
-            raise OperationError(space.w_IndexError, space.wrap("wrong index"))
 
     def getitem(self, space, index_list):
         return self.implementation.getitem_index(space, index_list)
@@ -199,6 +218,10 @@
         except ArrayArgumentException:
             self.setitem_array_int(space, w_idx, w_value)
 
+    def descr_delitem(self, space, w_idx):
+        raise OperationError(space.w_ValueError, space.wrap(
+            "cannot delete array elements"))
+
     def descr_len(self, space):
         shape = self.get_shape()
         if len(shape):
@@ -288,22 +311,7 @@
                     space.wrap('array does not have imaginary part to set'))
         self.implementation.set_imag(space, self, w_value)
 
-    def descr_reshape(self, space, args_w):
-        """reshape(...)
-        a.reshape(shape)
-
-        Returns an array containing the same data with a new shape.
-
-        Refer to `numpypy.reshape` for full documentation.
-
-        See Also
-        --------
-        numpypy.reshape : equivalent function
-        """
-        if len(args_w) == 1:
-            w_shape = args_w[0]
-        else:
-            w_shape = space.newtuple(args_w)
+    def reshape(self, space, w_shape):
         new_shape = get_shape_from_iterable(space, self.get_size(), w_shape)
         new_impl = self.implementation.reshape(space, self, new_shape)
         if new_impl is not None:
@@ -318,11 +326,42 @@
             arr.implementation.shape = new_shape
         return arr
 
+    def descr_reshape(self, space, __args__):
+        """reshape(...)
+        a.reshape(shape)
+
+        Returns an array containing the same data with a new shape.
+
+        Refer to `numpypy.reshape` for full documentation.
+
+        See Also
+        --------
+        numpypy.reshape : equivalent function
+        """
+        args_w, kw_w = __args__.unpack()
+        order = 'C'
+        if kw_w:
+            if "order" in kw_w:
+                order = space.str_w(kw_w["order"])
+                del kw_w["order"]
+            if kw_w:
+                raise OperationError(space.w_TypeError, space.wrap(
+                    "reshape() got unexpected keyword argument(s)"))
+        if order != 'C':
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "order not implemented"))
+        if len(args_w) == 1:
+            w_shape = args_w[0]
+        else:
+            w_shape = space.newtuple(args_w)
+        return self.reshape(space, w_shape)
+
     def descr_get_transpose(self, space):
         return W_NDimArray(self.implementation.transpose(self))
 
     def descr_transpose(self, space, args_w):
-        if len(args_w) != 0:
+        if not (len(args_w) == 0 or
+                len(args_w) == 1 and space.is_none(args_w[0])):
             raise OperationError(space.w_NotImplementedError, space.wrap(
                 "axes unsupported for transpose"))
         return self.descr_get_transpose(space)
@@ -364,7 +403,7 @@
         if order != 'C':
             raise OperationError(space.w_NotImplementedError, space.wrap(
                 "order not implemented"))
-        return self.descr_reshape(space, [space.wrap(-1)])
+        return self.reshape(space, space.wrap(-1))
 
     @unwrap_spec(w_axis=WrappedDefault(None),
                  w_out=WrappedDefault(None),
@@ -378,14 +417,14 @@
                                  space.wrap("axis unsupported for compress"))
             arr = self
         else:
-            arr = self.descr_reshape(space, [space.wrap(-1)])
+            arr = self.reshape(space, space.wrap(-1))
         index = convert_to_array(space, w_obj)
         return arr.getitem_filter(space, index)
 
     def descr_flatten(self, space, w_order=None):
         if self.is_scalar():
             # scalars have no storage
-            return self.descr_reshape(space, [space.wrap(1)])
+            return self.reshape(space, space.wrap(1))
         w_res = self.descr_ravel(space, w_order)
         if w_res.implementation.storage == self.implementation.storage:
             return w_res.descr_copy(space)
@@ -820,7 +859,7 @@
             # numpy compatability
             return W_NDimArray.new_scalar(space, dtype, space.wrap(0))
         # Do the dims match?
-        out_shape, other_critical_dim = match_dot_shapes(space, self, other)
+        out_shape, other_critical_dim = _match_dot_shapes(space, self, other)
         w_res = W_NDimArray.from_shape(space, out_shape, dtype, w_instance=self)
         # This is the place to add fpypy and blas
         return loop.multidim_dot(space, self, other,  w_res, dtype,
@@ -838,7 +877,7 @@
     # ----------------------- reduce -------------------------------
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False,
-                           cumultative=False):
+                           cumulative=False):
         def impl(self, space, w_axis=None, w_dtype=None, w_out=None):
             if space.is_none(w_out):
                 out = None
@@ -849,9 +888,9 @@
                 out = w_out
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(
                 space, self, promote_to_largest, w_axis,
-                False, out, w_dtype, cumultative=cumultative)
+                False, out, w_dtype, cumulative=cumulative)
         return func_with_new_name(impl, "reduce_%s_impl_%d_%d" % (ufunc_name,
-                    promote_to_largest, cumultative))
+                    promote_to_largest, cumulative))
 
     descr_sum = _reduce_ufunc_impl("add")
     descr_sum_promote = _reduce_ufunc_impl("add", True)
@@ -861,8 +900,8 @@
     descr_all = _reduce_ufunc_impl('logical_and')
     descr_any = _reduce_ufunc_impl('logical_or')
 
-    descr_cumsum = _reduce_ufunc_impl('add', cumultative=True)
-    descr_cumprod = _reduce_ufunc_impl('multiply', cumultative=True)
+    descr_cumsum = _reduce_ufunc_impl('add', cumulative=True)
+    descr_cumprod = _reduce_ufunc_impl('multiply', cumulative=True)
 
     def _reduce_argmax_argmin_impl(op_name):
         def impl(self, space, w_axis=None, w_out=None):
@@ -1046,6 +1085,7 @@
     __len__ = interp2app(W_NDimArray.descr_len),
     __getitem__ = interp2app(W_NDimArray.descr_getitem),
     __setitem__ = interp2app(W_NDimArray.descr_setitem),
+    __delitem__ = interp2app(W_NDimArray.descr_delitem),
 
     __repr__ = interp2app(W_NDimArray.descr_repr),
     __str__ = interp2app(W_NDimArray.descr_str),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -9,6 +9,7 @@
 from pypy.module.micronumpy.interp_support import unwrap_axis_arg
 from pypy.module.micronumpy.strides import shape_agreement
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
+from pypy.module.micronumpy.constants import *
 
 def done_if_true(dtype, val):
     return dtype.itemtype.bool(val)
@@ -86,7 +87,7 @@
             out = w_out
         return self.reduce(space, w_obj, False, #do not promote_to_largest
                     w_axis, True, #keepdims must be true
-                    out, w_dtype, cumultative=True)
+                    out, w_dtype, cumulative=True)
 
     @unwrap_spec(skipna=bool, keepdims=bool)
     def descr_reduce(self, space, w_obj, w_axis=None, w_dtype=None,
@@ -158,7 +159,7 @@
                            w_dtype)
 
     def reduce(self, space, w_obj, promote_to_largest, w_axis,
-               keepdims=False, out=None, dtype=None, cumultative=False):
+               keepdims=False, out=None, dtype=None, cumulative=False):
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
                 "supported for binary functions"))
@@ -192,7 +193,7 @@
                                 "%s.reduce without identity", self.name)
         if shapelen > 1 and axis < shapelen:
             temp = None
-            if cumultative:
+            if cumulative:
                 shape = obj_shape[:]
                 temp_shape = obj_shape[:axis] + obj_shape[axis + 1:]
                 if out:
@@ -226,15 +227,15 @@
             else:
                 out = W_NDimArray.from_shape(space, shape, dtype, w_instance=obj)
             return loop.do_axis_reduce(shape, self.func, obj, dtype, axis, out,
-                                       self.identity, cumultative, temp)
-        if cumultative:
+                                       self.identity, cumulative, temp)
+        if cumulative:
             if out:
                 if out.get_shape() != [obj.get_size()]:
                     raise OperationError(space.w_ValueError, space.wrap(
                         "out of incompatible size"))
             else:
                 out = W_NDimArray.from_shape(space, [obj.get_size()], dtype, w_instance=obj)
-            loop.compute_reduce_cumultative(obj, out, dtype, self.func,
+            loop.compute_reduce_cumulative(obj, out, dtype, self.func,
                                             self.identity)
             return out
         if out:
@@ -431,16 +432,16 @@
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
     # Some operations promote op(bool, bool) to return int8, rather than bool
-    if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
+    if promote_bools and (dt1.kind == dt2.kind == NPY_GENBOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
 
     # Everything numeric promotes to complex
     if dt2.is_complex_type() or dt1.is_complex_type():
-        if dt2.num == 14:
+        if dt2.num == NPY_CFLOAT:
             return interp_dtype.get_dtype_cache(space).w_complex64dtype
-        elif dt2.num == 15:
+        elif dt2.num == NPY_CDOUBLE:
             return interp_dtype.get_dtype_cache(space).w_complex128dtype
-        elif dt2.num == 16:
+        elif dt2.num == NPY_CLONGDOUBLE:
             return interp_dtype.get_dtype_cache(space).w_complexlongdtype
         else:
             raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
@@ -452,35 +453,30 @@
         return dt2
 
     # Everything promotes to float, and bool promotes to everything.
-    if dt2.kind == interp_dtype.FLOATINGLTR or dt1.kind == interp_dtype.BOOLLTR:
+    if dt2.kind == NPY_FLOATINGLTR or dt1.kind == NPY_GENBOOLLTR:
         # Float32 + 8-bit int = Float64
-        if dt2.num == 11 and dt1.itemtype.get_element_size() >= 4:
+        if dt2.num == NPY_FLOAT and dt1.itemtype.get_element_size() >= 4:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
         return dt2
 
     # for now this means mixing signed and unsigned
-    if dt2.kind == interp_dtype.SIGNEDLTR:
+    if dt2.kind == NPY_SIGNEDLTR:
         # if dt2 has a greater number of bytes, then just go with it
         if dt1.itemtype.get_element_size() < dt2.itemtype.get_element_size():
             return dt2
         # we need to promote both dtypes
         dtypenum = dt2.num + 2
-    elif dt2.num == 10 or (LONG_BIT == 64 and dt2.num == 8):
+    elif dt2.num == NPY_ULONGLONG or (LONG_BIT == 64 and dt2.num == NPY_ULONG):
         # UInt64 + signed = Float64
-        dtypenum = 12
+        dtypenum = NPY_DOUBLE
     elif dt2.is_flexible_type():
         # For those operations that get here (concatenate, stack),
         # flexible types take precedence over numeric type
         if dt2.is_record_type():
             return dt2
         if dt1.is_str_or_unicode():
-            if dt2.num == 18:
-                if dt2.itemtype.get_element_size() >= \
-                           dt1.itemtype.get_element_size():
-                    return dt2
-                return dt1
             if dt2.itemtype.get_element_size() >= \
-                       dt1.itemtype.get_element_size():
+                    dt1.itemtype.get_element_size():
                 return dt2
             return dt1
         return dt2
@@ -490,7 +486,7 @@
     newdtype = interp_dtype.get_dtype_cache(space).dtypes_by_num[dtypenum]
 
     if (newdtype.itemtype.get_element_size() > dt2.itemtype.get_element_size() or
-        newdtype.kind == interp_dtype.FLOATINGLTR):
+        newdtype.kind == NPY_FLOATINGLTR):
         return newdtype
     else:
         # we only promoted to long on 32-bit or to longlong on 64-bit
@@ -501,23 +497,23 @@
 @jit.unroll_safe
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
         promote_bools=False, promote_to_largest=False):
-    if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
+    if promote_bools and (dt.kind == NPY_GENBOOLLTR):
         return interp_dtype.get_dtype_cache(space).w_int8dtype
     if promote_to_float:
-        if dt.kind == interp_dtype.FLOATINGLTR or dt.kind==interp_dtype.COMPLEXLTR:
+        if dt.kind == NPY_FLOATINGLTR or dt.kind == NPY_COMPLEXLTR:
             return dt
-        if dt.num >= 5:
+        if dt.num >= NPY_INT:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
         for bytes, dtype in interp_dtype.get_dtype_cache(space).float_dtypes_by_num_bytes:
-            if (dtype.kind == interp_dtype.FLOATINGLTR and
+            if (dtype.kind == NPY_FLOATINGLTR and
                 dtype.itemtype.get_element_size() > dt.itemtype.get_element_size()):
                 return dtype
     if promote_to_largest:
-        if dt.kind == interp_dtype.BOOLLTR or dt.kind == interp_dtype.SIGNEDLTR:
+        if dt.kind == NPY_GENBOOLLTR or dt.kind == NPY_SIGNEDLTR:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
-        elif dt.kind == interp_dtype.FLOATINGLTR:
+        elif dt.kind == NPY_FLOATINGLTR:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
-        elif dt.kind == interp_dtype.UNSIGNEDLTR:
+        elif dt.kind == NPY_UNSIGNEDLTR:
             return interp_dtype.get_dtype_cache(space).w_uint64dtype
         else:
             assert False
@@ -559,8 +555,8 @@
         if (current_guess is None):
             return interp_dtype.variable_dtype(space,
                                                'S%d' % space.len_w(w_obj))
-        elif current_guess.num ==18:
-            if  current_guess.itemtype.get_size() < space.len_w(w_obj):
+        elif current_guess.num == NPY_STRING:
+            if current_guess.itemtype.get_size() < space.len_w(w_obj):
                 return interp_dtype.variable_dtype(space,
                                                    'S%d' % space.len_w(w_obj))
         return current_guess
diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py
--- a/pypy/module/micronumpy/iter.py
+++ b/pypy/module/micronumpy/iter.py
@@ -275,11 +275,11 @@
         return self.indexes[d]
 
 class AxisIterator(base.BaseArrayIterator):
-    def __init__(self, array, shape, dim, cumultative):
+    def __init__(self, array, shape, dim, cumulative):
         self.shape = shape
         strides = array.get_strides()
         backstrides = array.get_backstrides()
-        if cumultative:
+        if cumulative:
             self.strides = strides
             self.backstrides = backstrides
         elif len(shape) == len(strides):
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -10,8 +10,8 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.iter import PureShapeIterator
-from pypy.module.micronumpy import constants
-from pypy.module.micronumpy.support import int_w
+from pypy.module.micronumpy.support import index_w
+from pypy.module.micronumpy.constants import *
 
 call2_driver = jit.JitDriver(name='numpy_call2',
                              greens = ['shapelen', 'func', 'calc_dtype',
@@ -159,10 +159,16 @@
                                   greens = ['shapelen', 'func', 'dtype'],
                                   reds = 'auto')
 
-def compute_reduce_cumultative(obj, out, calc_dtype, func, identity):
+def compute_reduce_cumulative(obj, out, calc_dtype, func, identity):
     obj_iter = obj.create_iter()
     out_iter = out.create_iter()
-    cur_value = identity.convert_to(calc_dtype)
+    if identity is None:
+        cur_value = obj_iter.getitem().convert_to(calc_dtype)
+        out_iter.setitem(cur_value)
+        out_iter.next()
+        obj_iter.next()
+    else:
+        cur_value = identity.convert_to(calc_dtype)
     shapelen = len(obj.get_shape())
     while not obj_iter.done():
         reduce_cum_driver.jit_merge_point(shapelen=shapelen, func=func,
@@ -218,10 +224,10 @@
                                             'func', 'dtype'],
                                     reds='auto')
 
-def do_axis_reduce(shape, func, arr, dtype, axis, out, identity, cumultative,
+def do_axis_reduce(shape, func, arr, dtype, axis, out, identity, cumulative,
                    temp):
-    out_iter = out.create_axis_iter(arr.get_shape(), axis, cumultative)
-    if cumultative:
+    out_iter = out.create_axis_iter(arr.get_shape(), axis, cumulative)
+    if cumulative:
         temp_iter = temp.create_axis_iter(arr.get_shape(), axis, False)
     else:
         temp_iter = out_iter # hack
@@ -240,7 +246,7 @@
             cur = temp_iter.getitem()
             w_val = func(dtype, cur, w_val)
         out_iter.setitem(w_val)
-        if cumultative:
+        if cumulative:
             temp_iter.setitem(w_val)
             temp_iter.next()
         arr_iter.next()
@@ -581,15 +587,15 @@
     while not arr_iter.done():
         choose_driver.jit_merge_point(shapelen=shapelen, dtype=dtype,
                                       mode=mode)
-        index = int_w(space, arr_iter.getitem())
+        index = index_w(space, arr_iter.getitem())
         if index < 0 or index >= len(iterators):
-            if mode == constants.MODE_RAISE:
+            if mode == NPY_RAISE:
                 raise OperationError(space.w_ValueError, space.wrap(
                     "invalid entry in choice array"))
-            elif mode == constants.MODE_WRAP:
+            elif mode == NPY_WRAP:
                 index = index % (len(iterators))
             else:
-                assert mode == constants.MODE_CLIP
+                assert mode == NPY_CLIP
                 if index < 0:
                     index = 0
                 else:
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,11 +1,15 @@
 from rpython.rlib import jit
 from pypy.interpreter.error import OperationError
 
-def int_w(space, w_obj):
+def index_w(space, w_obj):
     try:
         return space.int_w(space.index(w_obj))
     except OperationError:
-        return space.int_w(space.int(w_obj))
+        try:
+            return space.int_w(space.int(w_obj))
+        except OperationError:
+            raise OperationError(space.w_IndexError, space.wrap(
+                "cannot convert index to integer"))
 
 @jit.unroll_safe
 def product(s):
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,7 +1,4 @@
-from pypy.module.micronumpy.interp_dtype import get_dtype_cache
-from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
-        find_unaryop_result_dtype)
-from pypy.module.micronumpy.interp_dtype import NATBYTE, OPPBYTE
+from pypy.module.micronumpy.interp_dtype import NPY_NATBYTE, NPY_OPPBYTE
 from pypy.conftest import option
 import sys
 
@@ -10,82 +7,9 @@
 
     @classmethod
     def setup_class(cls):
-        isNumpy = False
         if option.runappdirect:
             if '__pypy__' not in sys.builtin_module_names:
                 import numpy
                 sys.modules['numpypy'] = numpy
-                isNumpy = True
-        cls.w_isNumpy = cls.space.wrap(isNumpy)
-        cls.w_non_native_prefix = cls.space.wrap(OPPBYTE)
-        cls.w_native_prefix = cls.space.wrap(NATBYTE)
-
-class TestUfuncCoerscion(object):
-    def test_binops(self, space):
-        bool_dtype = get_dtype_cache(space).w_booldtype
-        int8_dtype = get_dtype_cache(space).w_int8dtype
-        int32_dtype = get_dtype_cache(space).w_int32dtype
-        float64_dtype = get_dtype_cache(space).w_float64dtype
-
-        # Basic pairing
-        assert find_binop_result_dtype(space, bool_dtype, bool_dtype) is bool_dtype
-        assert find_binop_result_dtype(space, bool_dtype, float64_dtype) is float64_dtype
-        assert find_binop_result_dtype(space, float64_dtype, bool_dtype) is float64_dtype
-        assert find_binop_result_dtype(space, int32_dtype, int8_dtype) is int32_dtype
-        assert find_binop_result_dtype(space, int32_dtype, bool_dtype) is int32_dtype
-
-        # With promote bool (happens on div), the result is that the op should
-        # promote bools to int8
-        assert find_binop_result_dtype(space, bool_dtype, bool_dtype, promote_bools=True) is int8_dtype
-        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_bools=True) is float64_dtype
-
-        # Coerce to floats
-        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_to_float=True) is float64_dtype
-
-    def test_unaryops(self, space):
-        bool_dtype = get_dtype_cache(space).w_booldtype
-        int8_dtype = get_dtype_cache(space).w_int8dtype
-        uint8_dtype = get_dtype_cache(space).w_uint8dtype
-        int16_dtype = get_dtype_cache(space).w_int16dtype
-        uint16_dtype = get_dtype_cache(space).w_uint16dtype
-        int32_dtype = get_dtype_cache(space).w_int32dtype
-        uint32_dtype = get_dtype_cache(space).w_uint32dtype
-        long_dtype = get_dtype_cache(space).w_longdtype
-        ulong_dtype = get_dtype_cache(space).w_ulongdtype
-        int64_dtype = get_dtype_cache(space).w_int64dtype
-        uint64_dtype = get_dtype_cache(space).w_uint64dtype
-        float16_dtype = get_dtype_cache(space).w_float16dtype
-        float32_dtype = get_dtype_cache(space).w_float32dtype
-        float64_dtype = get_dtype_cache(space).w_float64dtype
-
-        # Normal rules, everything returns itself
-        assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
-        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
-        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
-        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
-        assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
-        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
-        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
-        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
-        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
-        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
-        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
-
-        # Coerce to floats, some of these will eventually be float16, or
-        # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float16_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float16_dtype
-        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float16_dtype
-        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
-        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
-        assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
-
-        # promote bools, happens with sign ufunc
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
+        cls.w_non_native_prefix = cls.space.wrap(NPY_OPPBYTE)
+        cls.w_native_prefix = cls.space.wrap(NPY_NATBYTE)
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -34,6 +34,14 @@
 
         assert dtype(None) is dtype(float)
 
+        e = dtype('int8')
+        exc = raises(KeyError, "e[2]")
+        assert exc.value.message == "There are no fields in dtype int8."
+        exc = raises(KeyError, "e['z']")
+        assert exc.value.message == "There are no fields in dtype int8."
+        exc = raises(KeyError, "e[None]")
+        assert exc.value.message == "There are no fields in dtype int8."
+
         exc = raises(TypeError, dtype, (1, 2))
         assert 'data type not understood' in str(exc.value)
         raises(KeyError, 'dtype(int)["asdasd"]')
@@ -48,8 +56,21 @@
 
     def test_dtype_aliases(self):
         from numpypy import dtype
+        assert dtype('bool8') is dtype('bool')
+        assert dtype('byte') is dtype('int8')
+        assert dtype('ubyte') is dtype('uint8')
+        assert dtype('short') is dtype('int16')
+        assert dtype('ushort') is dtype('uint16')
+        assert dtype('longlong') is dtype('q')
+        assert dtype('ulonglong') is dtype('Q')
+        assert dtype("float") is dtype(float)
+        assert dtype('single') is dtype('float32')
+        assert dtype('double') is dtype('float64')
         assert dtype('longfloat').num in (12, 13)
         assert dtype('longdouble').num in (12, 13)
+        assert dtype('csingle') is dtype('complex64')
+        assert dtype('cfloat') is dtype('complex128')
+        assert dtype('cdouble') is dtype('complex128')
         assert dtype('clongfloat').num in (15, 16)
         assert dtype('clongdouble').num in (15, 16)
 
@@ -215,10 +236,6 @@
             pass
         assert True
 
-    def test_aliases(self):
-        from numpypy import dtype
-        assert dtype("float") is dtype(float)
-
     def test_index(self):
         import numpypy as np
         for dtype in [np.int8, np.int16, np.int32, np.int64]:
@@ -625,6 +642,11 @@
             assert numpy.intp is numpy.int64
             assert numpy.uintp is numpy.uint64
 
+        assert issubclass(numpy.float64, numpy.floating)
+        assert issubclass(numpy.longfloat, numpy.floating)
+        assert not issubclass(numpy.float64, numpy.longfloat)
+        assert not issubclass(numpy.longfloat, numpy.float64)
+
     def test_mro(self):
         import numpypy as numpy
 
@@ -828,7 +850,17 @@
         assert d["x"].itemsize == 16
         e = dtype([("x", "float", 2), ("y", "int", 2)])
         assert e.fields.keys() == keys
-        assert e['x'].shape == (2,)
+        for v in ['x', u'x', 0, -2]:
+            assert e[v] == (dtype('float'), (2,))
+        for v in ['y', u'y', 1, -1]:
+            assert e[v] == (dtype('int'), (2,))
+        for v in [-3, 2]:
+            exc = raises(IndexError, "e[%d]" % v)
+            assert exc.value.message == "Field index %d out of range." % v
+        exc = raises(KeyError, "e['z']")
+        assert exc.value.message == "Field named 'z' not found."
+        exc = raises(ValueError, "e[None]")
+        assert exc.value.message == 'Field key must be an integer, string, or unicode.'
 
         dt = dtype((float, 10))
         assert dt.shape == (10,)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -511,6 +511,12 @@
         a[self.CustomIntObject(1)] = 100
         assert a[1] == 100
 
+    def test_delitem(self):
+        import numpypy as np
+        a = np.arange(10)
+        exc = raises(ValueError, 'del a[2]')
+        assert exc.value.message == 'cannot delete array elements'
+
     def test_access_swallow_exception(self):
         class ErrorIndex(object):
             def __index__(self):
@@ -525,8 +531,10 @@
 
         from numpypy import arange
         a = arange(10)
-        raises(IndexError, "a[ErrorIndex()] == 0")
-        raises(IndexError, "a[ErrorInt()] == 0")
+        exc = raises(IndexError, "a[ErrorIndex()] == 0")
+        assert exc.value.message == 'cannot convert index to integer'
+        exc = raises(IndexError, "a[ErrorInt()] == 0")
+        assert exc.value.message == 'cannot convert index to integer'
 
     def test_setslice_array(self):
         from numpypy import array
@@ -696,6 +704,11 @@
         (a + a).reshape(2, 12) # assert did not explode
         a = array([[[[]]]])
         assert a.reshape((0,)).shape == (0,)
+        assert a.reshape((0,), order='C').shape == (0,)
+        raises(TypeError, a.reshape, (0,), badarg="C")
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.reshape, (0,), order='F')
 
     def test_slice_reshape(self):
         from numpypy import zeros, arange
@@ -994,6 +1007,13 @@
         b = a // 2
         assert (b == [0, 0, 1, 1, 2]).all()
 
+    def test_signed_integer_division_overflow(self):
+        import numpypy as np
+        for s in (8, 16, 32, 64):
+            for o in ['__div__', '__floordiv__']:
+                a = np.array([-2**(s-1)], dtype='int%d' % s)
+                assert getattr(a, o)(-1) == 0
+
     def test_truediv(self):
         from operator import truediv
         from numpypy import arange
@@ -1335,7 +1355,9 @@
         assert a.argmax() == 5
         assert a.argmax(axis=None, out=None) == 5
         assert a[:2, ].argmax() == 3
-        raises(NotImplementedError, a.argmax, axis=0)
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.argmax, axis=0)
 
     def test_argmin(self):
         from numpypy import array
@@ -1344,7 +1366,9 @@
         assert a.argmin(axis=None, out=None) == 3
         b = array([])
         raises(ValueError, "b.argmin()")
-        raises(NotImplementedError, a.argmin, axis=0)
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.argmin, axis=0)
 
     def test_all(self):
         from numpypy import array
@@ -1779,6 +1803,15 @@
         raises(IndexError, "arange(10)[array([10])] = 3")
         raises(IndexError, "arange(10)[[-11]] = 3")
 
+    def test_bool_single_index(self):
+        import numpypy as np
+        a = np.array([[1, 2, 3],
+                      [4, 5, 6],
+                      [7, 8, 9]])
+        a[np.array(True)]; skip("broken")  # check for crash but skip rest of test until correct
+        assert (a[np.array(True)] == a[1]).all()
+        assert (a[np.array(False)] == a[0]).all()
+
     def test_bool_array_index(self):
         from numpypy import arange, array
         b = arange(10)
@@ -2190,7 +2223,10 @@
         b = a.T
         assert(b[:, 0] == a[0, :]).all()
         assert (a.transpose() == b).all()
-        raises(NotImplementedError, a.transpose, (1, 0, 2))
+        assert (a.transpose(None) == b).all()
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, a.transpose, (1, 0, 2))
 
     def test_flatiter(self):
         from numpypy import array, flatiter, arange, zeros
@@ -2915,10 +2951,22 @@
         d = dtype([("x", "int", 3), ("y", "float", 5)])
         a = array([([1, 2, 3], [0.5, 1.5, 2.5, 3.5, 4.5]), ([4, 5, 6], [5.5, 6.5, 7.5, 8.5, 9.5])], dtype=d)
 
-        assert (a[0]["x"] == [1, 2, 3]).all()
-        assert (a[0]["y"] == [0.5, 1.5, 2.5, 3.5, 4.5]).all()
-        assert (a[1]["x"] == [4, 5, 6]).all()
-        assert (a[1]["y"] == [5.5, 6.5, 7.5, 8.5, 9.5]).all()
+        for v in ['x', u'x', 0, -2]:
+            assert (a[0][v] == [1, 2, 3]).all()
+            assert (a[1][v] == [4, 5, 6]).all()
+        for v in ['y', u'y', -1, 1]:
+            assert (a[0][v] == [0.5, 1.5, 2.5, 3.5, 4.5]).all()
+            assert (a[1][v] == [5.5, 6.5, 7.5, 8.5, 9.5]).all()
+        for v in [-3, 2]:
+            exc = raises(IndexError, "a[0][%d]" % v)
+            assert exc.value.message == "invalid index (%d)" % (v + 2 if v < 0 else v)
+        exc = raises(IndexError, "a[0]['z']")
+        assert exc.value.message == "invalid index"
+        exc = raises(IndexError, "a[0][None]")
+        assert exc.value.message == "invalid index"
+
+        exc = raises(IndexError, "a[0][None]")
+        assert exc.value.message == 'invalid index'
 
         a[0]["x"][0] = 200
         assert a[0]["x"][0] == 200
@@ -2939,6 +2987,8 @@
         a[0, 0] = 500
         assert (a[0, 0, 0] == 500).all()
         assert a[0, 0, 0].shape == (10,)
+        exc = raises(ValueError, "a[0, 0]['z']")
+        assert exc.value.message == 'field named z not found'
 
     def test_subarray_multiple_rows(self):
         import numpypy as np
diff --git a/pypy/module/micronumpy/test/test_scalar.py b/pypy/module/micronumpy/test/test_scalar.py
--- a/pypy/module/micronumpy/test/test_scalar.py
+++ b/pypy/module/micronumpy/test/test_scalar.py
@@ -3,6 +3,21 @@
 class AppTestScalar(BaseNumpyAppTest):
     spaceconfig = dict(usemodules=["micronumpy", "binascii", "struct"])
 
+    def test_init(self):
+        import numpypy as np
+        import math
+        assert np.intp() == np.intp(0)
+        assert np.intp('123') == np.intp(123)
+        raises(TypeError, np.intp, None)
+        assert np.float64() == np.float64(0)
+        assert math.isnan(np.float64(None))
+        assert np.bool_() == np.bool_(False)
+        assert np.bool_('abc') == np.bool_(True)
+        assert np.bool_(None) == np.bool_(False)
+        assert np.complex_() == np.complex_(0)
+        #raises(TypeError, np.complex_, '1+2j')
+        assert math.isnan(np.complex_(None))
+
     def test_pickle(self):
         from numpypy import dtype, int32, float64, complex128, zeros, sum
         from numpypy.core.multiarray import scalar
@@ -23,19 +38,19 @@
         assert loads(dumps(sum(a))) == sum(a)
 
     def test_round(self):
-        from numpypy import int32, float64, complex128, bool
+        from numpypy import int32, float64, complex128, bool_
         i = int32(1337)
         f = float64(13.37)
         c = complex128(13 + 37.j)
-        b = bool(0)
+        b = bool_(1)
         assert i.round(decimals=-2) == 1300
         assert i.round(decimals=1) == 1337
         assert c.round() == c
         assert f.round() == 13.
         assert f.round(decimals=-1) == 10.
         assert f.round(decimals=1) == 13.4
-        exc = raises(AttributeError, 'b.round()')
-        assert exc.value[0] == "'bool' object has no attribute 'round'"
+        assert b.round() == 1.0
+        assert b.round(decimals=5) is b
 
     def test_attributes(self):
         import numpypy as np
diff --git a/pypy/module/micronumpy/test/test_sorting.py b/pypy/module/micronumpy/test/test_sorting.py
--- a/pypy/module/micronumpy/test/test_sorting.py
+++ b/pypy/module/micronumpy/test/test_sorting.py
@@ -18,7 +18,9 @@
 
             a = arange(100, dtype=dtype)
             assert (a.argsort() == a).all()
-        raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
 
     def test_argsort_ndim(self):
         from numpypy import array
@@ -76,8 +78,10 @@
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
             b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
             c = a.copy()
-            exc = raises(NotImplementedError, a.sort)
-            assert exc.value[0].find('supported') >= 0
+            import sys
+            if '__pypy__' in sys.builtin_module_names:
+                exc = raises(NotImplementedError, a.sort)
+                assert exc.value[0].find('supported') >= 0
             #assert (a == b).all(), \
             #    'a,orig,dtype %r,%r,%r' % (a,c,dtype)
 
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -222,6 +222,7 @@
         assert b.called_finalize == True
 
     def test___array__(self):
+        import sys
         from numpypy import ndarray, array, dtype
         class D(ndarray):
             def __new__(subtype, shape, dtype):
@@ -239,7 +240,7 @@
         a = C([2, 2], int)
         b = array(a)
         assert b.shape == (2, 2)
-        if not self.isNumpy:
+        if '__pypy__' in sys.builtin_module_names:
             assert b.id == 'subtype'
             assert isinstance(b, D)
         c = array(a, float)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,6 +1,78 @@
-from pypy.conftest import option
-from pypy.interpreter.gateway import interp2app
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
+        find_unaryop_result_dtype)
+from pypy.module.micronumpy.interp_dtype import get_dtype_cache
+
+
+class TestUfuncCoercion(object):
+    def test_binops(self, space):
+        bool_dtype = get_dtype_cache(space).w_booldtype
+        int8_dtype = get_dtype_cache(space).w_int8dtype
+        int32_dtype = get_dtype_cache(space).w_int32dtype
+        float64_dtype = get_dtype_cache(space).w_float64dtype
+
+        # Basic pairing
+        assert find_binop_result_dtype(space, bool_dtype, bool_dtype) is bool_dtype
+        assert find_binop_result_dtype(space, bool_dtype, float64_dtype) is float64_dtype
+        assert find_binop_result_dtype(space, float64_dtype, bool_dtype) is float64_dtype
+        assert find_binop_result_dtype(space, int32_dtype, int8_dtype) is int32_dtype
+        assert find_binop_result_dtype(space, int32_dtype, bool_dtype) is int32_dtype
+
+        # With promote bool (happens on div), the result is that the op should
+        # promote bools to int8
+        assert find_binop_result_dtype(space, bool_dtype, bool_dtype, promote_bools=True) is int8_dtype
+        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_bools=True) is float64_dtype
+
+        # Coerce to floats
+        assert find_binop_result_dtype(space, bool_dtype, float64_dtype, promote_to_float=True) is float64_dtype
+
+    def test_unaryops(self, space):
+        bool_dtype = get_dtype_cache(space).w_booldtype
+        int8_dtype = get_dtype_cache(space).w_int8dtype
+        uint8_dtype = get_dtype_cache(space).w_uint8dtype
+        int16_dtype = get_dtype_cache(space).w_int16dtype
+        uint16_dtype = get_dtype_cache(space).w_uint16dtype
+        int32_dtype = get_dtype_cache(space).w_int32dtype
+        uint32_dtype = get_dtype_cache(space).w_uint32dtype
+        long_dtype = get_dtype_cache(space).w_longdtype
+        ulong_dtype = get_dtype_cache(space).w_ulongdtype
+        int64_dtype = get_dtype_cache(space).w_int64dtype
+        uint64_dtype = get_dtype_cache(space).w_uint64dtype
+        float16_dtype = get_dtype_cache(space).w_float16dtype
+        float32_dtype = get_dtype_cache(space).w_float32dtype
+        float64_dtype = get_dtype_cache(space).w_float64dtype
+
+        # Normal rules, everything returns itself
+        assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
+        assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
+        assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
+        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
+        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
+
+        # Coerce to floats, some of these will eventually be float16, or
+        # whatever our smallest float type is.
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float16_dtype
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float16_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float16_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
+
+        # promote bools, happens with sign ufunc
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
 
 
 class AppTestUfuncs(BaseNumpyAppTest):
@@ -994,3 +1066,14 @@
         print b
         assert (b == [[0, 0, 1], [1, 3, 5]]).all()
         assert b.dtype == int
+
+    def test_noncommutative_reduce_accumulate(self):
+        import numpypy as np
+        tosubtract = np.arange(5)
+        todivide = np.array([2.0, 0.5, 0.25])
+        assert np.subtract.reduce(tosubtract) == -10
+        assert np.divide.reduce(todivide) == 16.0
+        assert (np.subtract.accumulate(tosubtract) ==
+                np.array([0, -1, -3, -6, -10])).all()
+        assert (np.divide.accumulate(todivide) ==
+                np.array([2., 4., 16.])).all()
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -115,6 +115,10 @@
 class BaseType(object):
     SortRepr = None # placeholders for sorting classes, overloaded in sort.py
     Sort = None
+    _immutable_fields_ = ['native']
+
+    def __init__(self, native=True):
+        self.native = native
 
     def _unimplemented_ufunc(self, *args):
         raise NotImplementedError
@@ -172,7 +176,15 @@
         raise NotImplementedError
 

From noreply at buildbot.pypy.org  Wed Oct 30 17:07:20 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 30 Oct 2013 17:07:20 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Tweak the jit counters: decay them
 at each minor collection. This
Message-ID: <20131030160720.35E421C1066@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67734:bc7ae175a924
Date: 2013-10-30 14:52 +0100
http://bitbucket.org/pypy/pypy/changeset/bc7ae175a924/

Log:	Tweak the jit counters: decay them at each minor collection. This
	requires refactorings to move them to some single table, but it
	should reduce total memory usage. This is done at the cost of hash
	collisions that might make them reach their bound earlier, but this
	should be rare and so completely unimportant.


From noreply at buildbot.pypy.org  Wed Oct 30 17:07:21 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 30 Oct 2013 17:07:21 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Add a jit parameter.
Message-ID: <20131030160721.844651C1066@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67735:f6296263d647
Date: 2013-10-30 14:55 +0100
http://bitbucket.org/pypy/pypy/changeset/f6296263d647/

Log:	Add a jit parameter.

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -442,6 +442,7 @@
     'threshold': 'number of times a loop has to run for it to become hot',
     'function_threshold': 'number of times a function must run for it to become traced from start',
     'trace_eagerness': 'number of times a guard has to fail before we start compiling a bridge',
+    'decay': 'decay counters at each minor collection: percentage kept',
     'trace_limit': 'number of recorded operations before we abort tracing with ABORT_TOO_LONG',
     'inlining': 'inline python functions or not (1/0)',
     'loop_longevity': 'a parameter controlling how long loops will be kept before being freed, an estimate',
@@ -455,6 +456,7 @@
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
               'function_threshold': 1619, # slightly more than one above, also prime
               'trace_eagerness': 200,
+              'decay': 90,
               'trace_limit': 6000,
               'inlining': 1,
               'loop_longevity': 1000,

From noreply at buildbot.pypy.org  Wed Oct 30 17:07:22 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 30 Oct 2013 17:07:22 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Should carefully have no effect:
 split the double-meaning of the 'counter'
Message-ID: <20131030160722.A90A21C1066@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67736:05ec1173e85d
Date: 2013-10-04 06:40 +0200
http://bitbucket.org/pypy/pypy/changeset/05ec1173e85d/

Log:	Should carefully have no effect: split the double-meaning of the
	'counter' on Cell objects in two fields. (transplanted from
	0fe3cbf3f18d85e1819c52c03e265bb4ee19f0e7)

diff --git a/rpython/jit/metainterp/test/test_warmstate.py b/rpython/jit/metainterp/test/test_warmstate.py
--- a/rpython/jit/metainterp/test/test_warmstate.py
+++ b/rpython/jit/metainterp/test/test_warmstate.py
@@ -4,6 +4,7 @@
 from rpython.jit.metainterp.warmstate import wrap, unwrap, specialize_value
 from rpython.jit.metainterp.warmstate import equal_whatever, hash_whatever
 from rpython.jit.metainterp.warmstate import WarmEnterState, JitCell
+from rpython.jit.metainterp.warmstate import MODE_HAVE_PROC, MODE_TRACING
 from rpython.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from rpython.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr
 from rpython.jit.codewriter import longlong
@@ -162,7 +163,7 @@
                                       constfloat(2.25)],
                                      looptoken)
     cell1 = get_jitcell(True, 5, 2.25)
-    assert cell1.counter < 0
+    assert cell1.mode == MODE_HAVE_PROC
     assert cell1.get_procedure_token() is looptoken
 
 def test_make_jitdriver_callbacks_1():
@@ -299,17 +300,17 @@
     #
     for i in range(1, 20005):
         cell = get_jitcell(True, i)
-        cell.counter = -1
+        cell.mode = MODE_HAVE_PROC
         cell.wref_procedure_token = None    # or a dead weakref, equivalently
         assert len(warmstate._jitcell_dict) == (i % 20000) + 1
     #
-    # Same test, with counter == -2 (rare case, kept alive)
+    # Same test, with mode == MODE_TRACING (rare case, kept alive)
     warmstate = WarmEnterState(None, FakeJitDriverSD())
     get_jitcell = warmstate._make_jitcell_getter_default()
     cell = get_jitcell(True, -1)
-    cell.counter = -2
+    cell.mode = MODE_TRACING
     #
     for i in range(1, 20005):
         cell = get_jitcell(True, i)
-        cell.counter = -2
+        cell.mode = MODE_TRACING
         assert len(warmstate._jitcell_dict) == i + 1
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -124,12 +124,13 @@
         return rffi.cast(lltype.Signed, x)
 
 
+MODE_COUNTING  = '\x00'   # not yet traced, wait till threshold is reached
+MODE_TRACING   = 'T'      # tracing is currently going on for this cell
+MODE_HAVE_PROC = 'P'      # there is an entry bridge for this cell
+
 class JitCell(BaseJitCell):
-    # the counter can mean the following things:
-    #     counter >=  0: not yet traced, wait till threshold is reached
-    #     counter == -1: there is an entry bridge for this cell
-    #     counter == -2: tracing is currently going on for this cell
-    counter = 0
+    counter = 0    # when THRESHOLD_LIMIT is reached, start tracing
+    mode = MODE_COUNTING
     dont_trace_here = False
     extra_delay = chr(0)
     wref_procedure_token = None
@@ -241,7 +242,7 @@
         cell = self.jit_cell_at_key(greenkey)
         old_token = cell.get_procedure_token()
         cell.set_procedure_token(procedure_token)
-        cell.counter = -1       # valid procedure bridge attached
+        cell.mode = MODE_HAVE_PROC       # valid procedure bridge attached
         if old_token is not None:
             self.cpu.redirect_call_assembler(old_token, procedure_token)
             # procedure_token is also kept alive by any loop that used
@@ -320,19 +321,19 @@
                 cell.extra_delay = curgen
                 return
             #
+            cell.counter = 0
             if not confirm_enter_jit(*args):
-                cell.counter = 0
                 return
             # start tracing
             from rpython.jit.metainterp.pyjitpl import MetaInterp
             metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
-            # set counter to -2, to mean "tracing in effect"
-            cell.counter = -2
+            cell.mode = MODE_TRACING
             try:
                 metainterp.compile_and_run_once(jitdriver_sd, *args)
             finally:
-                if cell.counter == -2:
+                if cell.mode == MODE_TRACING:
                     cell.counter = 0
+                    cell.mode = MODE_COUNTING
 
         def maybe_compile_and_run(threshold, *args):
             """Entry point to the JIT.  Called at the point with the
@@ -341,8 +342,9 @@
             # look for the cell corresponding to the current greenargs
             greenargs = args[:num_green_args]
             cell = get_jitcell(True, *greenargs)
+            mode = cell.mode
 
-            if cell.counter >= 0:
+            if mode == MODE_COUNTING:
                 # update the profiling counter
                 n = cell.counter + threshold
                 if n <= self.THRESHOLD_LIMIT:       # bound not reached
@@ -351,9 +353,10 @@
                 else:
                     bound_reached(cell, *args)
                     return
+
             else:
-                if cell.counter != -1:
-                    assert cell.counter == -2
+                if mode != MODE_HAVE_PROC:
+                    assert mode == MODE_TRACING
                     # tracing already happening in some outer invocation of
                     # this function. don't trace a second time.
                     return
@@ -363,6 +366,7 @@
                 procedure_token = cell.get_procedure_token()
                 if procedure_token is None:   # it was a weakref that has been freed
                     cell.counter = 0
+                    cell.mode = MODE_COUNTING
                     return
                 # extract and unspecialize the red arguments to pass to
                 # the assembler
@@ -459,11 +463,11 @@
             minimum = self.THRESHOLD_LIMIT // 20     # minimum 5%
             killme = []
             for key, cell in jitcell_dict.iteritems():
-                if cell.counter >= 0:
+                if cell.mode == MODE_COUNTING:
                     cell.counter = int(cell.counter * 0.92)
                     if cell.counter < minimum:
                         killme.append(key)
-                elif (cell.counter == -1
+                elif (cell.mode == MODE_HAVE_PROC
                       and cell.get_procedure_token() is None):
                     killme.append(key)
             for key in killme:
@@ -589,8 +593,11 @@
             procedure_token = cell.get_procedure_token()
             if procedure_token is None:
                 from rpython.jit.metainterp.compile import compile_tmp_callback
-                if cell.counter == -1:    # used to be a valid entry bridge,
-                    cell.counter = 0      # but was freed in the meantime.
+                if cell.mode == MODE_HAVE_PROC:
+                    # used to be a valid entry bridge,
+                    # but was freed in the meantime.
+                    cell.counter = 0
+                    cell.mode = MODE_COUNTING
                 memmgr = warmrunnerdesc.memory_manager
                 procedure_token = compile_tmp_callback(cpu, jd, greenkey,
                                                        redargtypes, memmgr)

From noreply at buildbot.pypy.org  Wed Oct 30 17:07:23 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 30 Oct 2013 17:07:23 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: in-progress
Message-ID: <20131030160723.E140F1C1066@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67737:375fd4fb9083
Date: 2013-10-30 17:06 +0100
http://bitbucket.org/pypy/pypy/changeset/375fd4fb9083/

Log:	in-progress

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/counter.py
@@ -0,0 +1,94 @@
+from rpython.rlib.rarithmetic import r_singlefloat
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+
+class JitCounter:
+    DEFAULT_SIZE = 4096
+
+    def __init__(self, size=DEFAULT_SIZE):
+        assert size >= 1 and (size & (size - 1)) == 0     # a power of two
+        self.mask = size - 1
+        self.timetable = lltype.malloc(rffi.CArray(rffi.FLOAT), size,
+                                       flavor='raw', zero=True,
+                                       track_allocation=False)
+        self.celltable = [None] * size
+
+    def compute_threshold(self, threshold):
+        """Return the 'increment' value corresponding to the given number."""
+        if threshold <= 0:
+            return 0.0   # no increment, never reach 1.0
+        if threshold < 2:
+            threshold = 2
+        return 1.0 / threshold   # the number is at most 0.5
+
+    def tick(self, hash, increment):
+        hash &= self.mask
+        counter = float(self.timetable[hash]) + increment
+        if counter < 1.0:
+            self.timetable[hash] = r_singlefloat(counter)
+            return False
+        else:
+            return True
+    tick._always_inline_ = True
+
+    def reset(self, hash):
+        hash &= self.mask
+        self.timetable[hash] = r_singlefloat(0.0)
+
+    def lookup_chain(self, hash):
+        hash &= self.mask
+        return self.celltable[hash]
+
+    def cleanup_chain(self, hash):
+        self.install_new_cell(hash, None)
+
+    def install_new_cell(self, hash, newcell):
+        hash &= self.mask
+        cell = self.celltable[hash]
+        keep = newcell
+        while cell is not None:
+            remove_me = cell.should_remove_jitcell()
+            nextcell = cell.next
+            if not remove_me:
+                cell.next = keep
+                keep = cell
+            cell = nextcell
+        self.celltable[hash] = keep
+
+    def set_decay(self, decay):
+        """Set the decay, from 0 (none) to 1000 (max)."""
+        if decay < 0:
+            decay = 0
+        elif decay > 1000:
+            decay = 1000
+        self.decay_by_mult = 1.0 - (decay * 0.001)
+
+    def decay_all_counters(self):
+        # Called during a minor collection by the GC, to gradually decay
+        # counters that didn't reach their maximum.  Thus if a counter
+        # is incremented very slowly, it will never reach the maximum.
+        # This avoids altogether the JIT compilation of rare paths.
+        # We also call this function when any maximum bound is reached,
+        # to avoid sudden bursts of JIT-compilation (the next one will
+        # not reach the maximum bound immmediately after).  This is
+        # important in corner cases where we would suddenly compile more
+        # than one loop because all counters reach the bound at the same
+        # time, but where compiling all but the first one is pointless.
+        size = self.mask + 1
+        pypy__decay_jit_counters(self.timetable, self.decay_by_mult, size)
+
+
+# this function is written directly in C; gcc will optimize it using SSE
+eci = ExternalCompilationInfo(post_include_bits=["""
+static void pypy__decay_jit_counters(float table[], double f1, long size1) {
+    float f = (float)f1;
+    int i, size = (int)size1;
+    for (i=0; i<size; i++)
+        table[i] *= f;
+}
+"""])
+
+pypy__decay_jit_counters = rffi.llexternal(
+    "pypy__decay_jit_counters", [rffi.FLOATP, lltype.Float, lltype.Signed],
+    lltype.Void, compilation_info=eci, _nowrapper=True, sandboxsafe=True)
diff --git a/rpython/jit/metainterp/test/test_counter.py b/rpython/jit/metainterp/test/test_counter.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/test/test_counter.py
@@ -0,0 +1,50 @@
+from rpython.jit.metainterp.counter import JitCounter
+
+
+def test_tick():
+    jc = JitCounter()
+    incr = jc.compute_threshold(4)
+    for i in range(5):
+        r = jc.tick(1234567, incr)
+        assert r is (i >= 3)
+    for i in range(5):
+        r = jc.tick(1234568, incr)
+        s = jc.tick(1234569, incr)
+        assert r is (i >= 3)
+        assert s is (i >= 3)
+    jc.reset(1234568)
+    for i in range(5):
+        r = jc.tick(1234568, incr)
+        s = jc.tick(1234569, incr)
+        assert r is (i >= 3)
+        assert s is True
+
+def test_install_new_chain():
+    class Dead:
+        next = None
+        def should_remove_jitcell(self):
+            return True
+    class Alive:
+        next = None
+        def should_remove_jitcell(self):
+            return False
+    #
+    jc = JitCounter()
+    assert jc.lookup_chain(1234567) is None
+    d1 = Dead()
+    jc.install_new_cell(1234567, d1)
+    assert jc.lookup_chain(1234567) is d1
+    d2 = Dead()
+    jc.install_new_cell(1234567, d2)
+    assert jc.lookup_chain(1234567) is d2
+    assert d2.next is None
+    #
+    d3 = Alive()
+    jc.install_new_cell(1234567, d3)
+    assert jc.lookup_chain(1234567) is d3
+    assert d3.next is None
+    d4 = Alive()
+    jc.install_new_cell(1234567, d4)
+    assert jc.lookup_chain(1234567) is d3
+    assert d3.next is d4
+    assert d4.next is None
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -205,6 +205,9 @@
         vrefinfo = VirtualRefInfo(self)
         self.codewriter.setup_vrefinfo(vrefinfo)
         #
+        from rpython.jit.metainterp.counter import JitCounter
+        self.jitcounter = JitCounter()
+        #
         self.hooks = policy.jithookiface
         self.make_virtualizable_infos()
         self.make_driverhook_graphs()
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -124,16 +124,11 @@
         return rffi.cast(lltype.Signed, x)
 
 
-MODE_COUNTING  = '\x00'   # not yet traced, wait till threshold is reached
-MODE_TRACING   = 'T'      # tracing is currently going on for this cell
-MODE_HAVE_PROC = 'P'      # there is an entry bridge for this cell
-
 class JitCell(BaseJitCell):
-    counter = 0    # when THRESHOLD_LIMIT is reached, start tracing
-    mode = MODE_COUNTING
-    dont_trace_here = False
-    extra_delay = chr(0)
+    tracing = False
+    dont_trace_here = chr(0)
     wref_procedure_token = None
+    next = None
 
     def get_procedure_token(self):
         if self.wref_procedure_token is not None:
@@ -149,6 +144,18 @@
         assert token is not None
         return weakref.ref(token)
 
+    def should_remove_jitcell(self):
+        if self.get_procedure_token() is not None:
+            return False    # don't remove JitCells with a procedure_token
+        if self.tracing:
+            return False    # don't remove JitCells that are being traced
+        if ord(self.dont_trace_here) == 0:
+            return True     # no reason to keep this JitCell
+        else:
+            # decrement dont_trace_here; it will eventually reach zero.
+            self.dont_trace_here = chr(ord(self.dont_trace_here) - 1)
+            return False
+
 # ____________________________________________________________
 
 
@@ -172,12 +179,7 @@
             meth(default_value)
 
     def _compute_threshold(self, threshold):
-        if threshold <= 0:
-            return 0 # never reach the THRESHOLD_LIMIT
-        if threshold < 2:
-            threshold = 2
-        return (self.THRESHOLD_LIMIT // threshold) + 1
-        # the number is at least 1, and at most about half THRESHOLD_LIMIT
+        return self.warmrunnerdesc.jitcounter.compute_threshold(threshold)
 
     def set_param_threshold(self, threshold):
         self.increment_threshold = self._compute_threshold(threshold)
@@ -186,11 +188,14 @@
         self.increment_function_threshold = self._compute_threshold(threshold)
 
     def set_param_trace_eagerness(self, value):
-        self.trace_eagerness = value
+        self.increment_trace_eagerness = self._compute_threshold(value)
 
     def set_param_trace_limit(self, value):
         self.trace_limit = value
 
+    def set_param_decay(self, decay):
+        self.warmrunnerdesc.jitcounter.set_decay(decay)
+
     def set_param_inlining(self, value):
         self.inlining = value
 
@@ -232,7 +237,7 @@
 
     def disable_noninlinable_function(self, greenkey):
         cell = self.jit_cell_at_key(greenkey)
-        cell.dont_trace_here = True
+        cell.dont_trace_here = chr(20)
         debug_start("jit-disableinlining")
         loc = self.get_location_str(greenkey)
         debug_print("disabled inlining", loc)
@@ -242,7 +247,6 @@
         cell = self.jit_cell_at_key(greenkey)
         old_token = cell.get_procedure_token()
         cell.set_procedure_token(procedure_token)
-        cell.mode = MODE_HAVE_PROC       # valid procedure bridge attached
         if old_token is not None:
             self.cpu.redirect_call_assembler(old_token, procedure_token)
             # procedure_token is also kept alive by any loop that used
@@ -281,6 +285,7 @@
                 assert 0, kind
         func_execute_token = self.cpu.make_execute_token(*ARGS)
         cpu = self.cpu
+        jitcounter = self.warmrunnerdesc.jitcounter
 
         def execute_assembler(loop_token, *args):
             # Call the backend to run the 'looptoken' with the given
@@ -306,34 +311,20 @@
             assert 0, "should have raised"
 
         def bound_reached(cell, *args):
-            # bound reached, but we do a last check: if it is the first
-            # time we reach the bound, or if another loop or bridge was
-            # compiled since the last time we reached it, then decrease
-            # the counter by a few percents instead.  It should avoid
-            # sudden bursts of JIT-compilation, and also corner cases
-            # where we suddenly compile more than one loop because all
-            # counters reach the bound at the same time, but where
-            # compiling all but the first one is pointless.
-            curgen = warmrunnerdesc.memory_manager.current_generation
-            curgen = chr(intmask(curgen) & 0xFF)    # only use 8 bits
-            if we_are_translated() and curgen != cell.extra_delay:
-                cell.counter = int(self.THRESHOLD_LIMIT * 0.98)
-                cell.extra_delay = curgen
-                return
-            #
+            jitcounter.reset(
             cell.counter = 0
             if not confirm_enter_jit(*args):
                 return
             # start tracing
             from rpython.jit.metainterp.pyjitpl import MetaInterp
             metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
-            cell.mode = MODE_TRACING
+            cell.tracing = True
+            cell.reset_counter()
             try:
                 metainterp.compile_and_run_once(jitdriver_sd, *args)
             finally:
-                if cell.mode == MODE_TRACING:
-                    cell.counter = 0
-                    cell.mode = MODE_COUNTING
+                cell.tracing = False
+                cell.reset_counter()
 
         def maybe_compile_and_run(threshold, *args):
             """Entry point to the JIT.  Called at the point with the
@@ -565,7 +556,7 @@
             if can_never_inline(*greenargs):
                 return False
             cell = jit_getter(False, *greenargs)
-            if cell is not None and cell.dont_trace_here:
+            if cell is not None and ord(cell.dont_trace_here) != 0:
                 return False
             return True
         def can_inline_callable(greenkey):
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -442,7 +442,7 @@
     'threshold': 'number of times a loop has to run for it to become hot',
     'function_threshold': 'number of times a function must run for it to become traced from start',
     'trace_eagerness': 'number of times a guard has to fail before we start compiling a bridge',
-    'decay': 'decay counters at each minor collection: percentage kept',
+    'decay': 'decay counters at each minor collection (0=none, 1000=max)',
     'trace_limit': 'number of recorded operations before we abort tracing with ABORT_TOO_LONG',
     'inlining': 'inline python functions or not (1/0)',
     'loop_longevity': 'a parameter controlling how long loops will be kept before being freed, an estimate',
@@ -456,7 +456,7 @@
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
               'function_threshold': 1619, # slightly more than one above, also prime
               'trace_eagerness': 200,
-              'decay': 90,
+              'decay': 100,
               'trace_limit': 6000,
               'inlining': 1,
               'loop_longevity': 1000,

From noreply at buildbot.pypy.org  Wed Oct 30 17:13:46 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 17:13:46 +0100 (CET)
Subject: [pypy-commit] pypy default: merge remove-numpypy
Message-ID: <20131030161346.8A9341C0330@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67738:f8579ebf6c24
Date: 2013-10-30 12:10 -0400
http://bitbucket.org/pypy/pypy/changeset/f8579ebf6c24/

Log:	merge remove-numpypy

diff too long, truncating to 2000 out of 7423 lines

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from . import core
-from .core import *
-from . import lib
-from .lib import *
-
-from __builtin__ import bool, int, long, float, complex, object, unicode, str
-
-from .core import round, abs, max, min
-
-__version__ = '1.7.0'
-
-__all__ = ['__version__']
-__all__ += core.__all__
-__all__ += lib.__all__
-
-#import sys
-#sys.modules.setdefault('numpy', sys.modules['numpypy'])
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from __future__ import division, absolute_import, print_function
-
-from . import multiarray
-from . import umath
-from . import numeric
-from .numeric import *
-from . import fromnumeric
-from .fromnumeric import *
-from . import shape_base
-from .shape_base import *
-
-from .fromnumeric import amax as max, amin as min, \
-    round_ as round
-from .numeric import absolute as abs
-
-__all__ = []
-__all__ += numeric.__all__
-__all__ += fromnumeric.__all__
-__all__ += shape_base.__all__
diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/_methods.py
+++ /dev/null
@@ -1,124 +0,0 @@
-"""
-Array methods which are called by the both the C-code for the method
-and the Python code for the NumPy-namespace function
-
-"""
-from __future__ import division, absolute_import, print_function
-
-import warnings
-
-from . import multiarray as mu
-from . import umath as um
-from .numeric import asanyarray
-from . import numerictypes as nt
-
-def _amax(a, axis=None, out=None, keepdims=False):
-    return um.maximum.reduce(a, axis=axis,
-                            out=out, keepdims=keepdims)
-
-def _amin(a, axis=None, out=None, keepdims=False):
-    return um.minimum.reduce(a, axis=axis,
-                            out=out, keepdims=keepdims)
-
-def _sum(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.add.reduce(a, axis=axis, dtype=dtype,
-                            out=out, keepdims=keepdims)
-
-def _prod(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.multiply.reduce(a, axis=axis, dtype=dtype,
-                            out=out, keepdims=keepdims)
-
-def _any(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.logical_or.reduce(a, axis=axis, dtype=dtype, out=out,
-                                keepdims=keepdims)
-
-def _all(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.logical_and.reduce(a, axis=axis, dtype=dtype, out=out,
-                                 keepdims=keepdims)
-
-def _count_reduce_items(arr, axis):
-    if axis is None:
-        axis = tuple(range(arr.ndim))
-    if not isinstance(axis, tuple):
-        axis = (axis,)
-    items = 1
-    for ax in axis:
-        items *= arr.shape[ax]
-    return items
-
-def _mean(a, axis=None, dtype=None, out=None, keepdims=False):
-    arr = asanyarray(a)
-
-    rcount = _count_reduce_items(arr, axis)
-    # Make this warning show up first
-    if rcount == 0:
-        warnings.warn("Mean of empty slice.", RuntimeWarning)
-
-
-    # Cast bool, unsigned int, and int to float64 by default
-    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
-        dtype = mu.dtype('f8')
-
-    ret = um.add.reduce(arr, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
-    if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(
-                ret, rcount, out=ret, casting='unsafe', subok=False)
-    else:
-        ret = ret.dtype.type(ret / rcount)
-
-    return ret
-
-def _var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
-    arr = asanyarray(a)
-
-    rcount = _count_reduce_items(arr, axis)
-    # Make this warning show up on top.
-    if ddof >= rcount:
-        warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning)
-
-    # Cast bool, unsigned int, and int to float64 by default
-    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
-        dtype = mu.dtype('f8')
-
-    # Compute the mean.
-    # Note that if dtype is not of inexact type then arraymean will
-    # not be either.
-    arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True)
-    if isinstance(arrmean, mu.ndarray):
-        arrmean = um.true_divide(
-                arrmean, rcount, out=arrmean, casting='unsafe', subok=False)
-    else:
-        arrmean = arrmean.dtype.type(arrmean / rcount)
-
-    # Compute sum of squared deviations from mean
-    # Note that x may not be inexact and that we need it to be an array,
-    # not a scalar.
-    x = asanyarray(arr - arrmean)
-    if issubclass(arr.dtype.type, nt.complexfloating):
-        x = um.multiply(x, um.conjugate(x), out=x).real
-    else:
-        x = um.multiply(x, x, out=x)
-    ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
-
-    # Compute degrees of freedom and make sure it is not negative.
-    rcount = max([rcount - ddof, 0])
-
-    # divide by degrees of freedom
-    if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(
-                ret, rcount, out=ret, casting='unsafe', subok=False)
-    else:
-        ret = ret.dtype.type(ret / rcount)
-
-    return ret
-
-def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
-    ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
-               keepdims=keepdims)
-
-    if isinstance(ret, mu.ndarray):
-        ret = um.sqrt(ret, out=ret)
-    else:
-        ret = ret.dtype.type(um.sqrt(ret))
-
-    return ret
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ /dev/null
@@ -1,751 +0,0 @@
-"""Array printing function
-
-$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $
-"""
-__all__ = ["array2string", "set_printoptions", "get_printoptions"]
-__docformat__ = 'restructuredtext'
-
-#
-# Written by Konrad Hinsen <hinsenk at ere.umontreal.ca>
-# last revision: 1996-3-13
-# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details)
-# and by Perry Greenfield 2000-4-1 for numarray
-# and by Travis Oliphant  2005-8-22 for numpy
-
-import sys
-import numerictypes as _nt
-from umath import maximum, minimum, absolute, not_equal, isnan, isinf
-#from multiarray import format_longfloat, datetime_as_string, datetime_data
-from fromnumeric import ravel
-
-
-def product(x, y): return x*y
-
-_summaryEdgeItems = 3     # repr N leading and trailing items of each dimension
-_summaryThreshold = 1000 # total items > triggers array summarization
-
-_float_output_precision = 8
-_float_output_suppress_small = False
-_line_width = 75
-_nan_str = 'nan'
-_inf_str = 'inf'
-_formatter = None  # formatting function for array elements
-
-if sys.version_info[0] >= 3:
-    from functools import reduce
-
-def set_printoptions(precision=None, threshold=None, edgeitems=None,
-                     linewidth=None, suppress=None,
-                     nanstr=None, infstr=None,
-                     formatter=None):
-    """
-    Set printing options.
-
-    These options determine the way floating point numbers, arrays and
-    other NumPy objects are displayed.
-
-    Parameters
-    ----------
-    precision : int, optional
-        Number of digits of precision for floating point output (default 8).
-    threshold : int, optional
-        Total number of array elements which trigger summarization
-        rather than full repr (default 1000).
-    edgeitems : int, optional
-        Number of array items in summary at beginning and end of
-        each dimension (default 3).
-    linewidth : int, optional
-        The number of characters per line for the purpose of inserting
-        line breaks (default 75).
-    suppress : bool, optional
-        Whether or not suppress printing of small floating point values
-        using scientific notation (default False).
-    nanstr : str, optional
-        String representation of floating point not-a-number (default nan).
-    infstr : str, optional
-        String representation of floating point infinity (default inf).
-    formatter : dict of callables, optional
-        If not None, the keys should indicate the type(s) that the respective
-        formatting function applies to.  Callables should return a string.
-        Types that are not specified (by their corresponding keys) are handled
-        by the default formatters.  Individual types for which a formatter
-        can be set are::
-
-            - 'bool'
-            - 'int'
-            - 'timedelta' : a `numpy.timedelta64`
-            - 'datetime' : a `numpy.datetime64`
-            - 'float'
-            - 'longfloat' : 128-bit floats
-            - 'complexfloat'
-            - 'longcomplexfloat' : composed of two 128-bit floats
-            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
-            - 'str' : all other strings
-
-        Other keys that can be used to set a group of types at once are::
-
-            - 'all' : sets all types
-            - 'int_kind' : sets 'int'
-            - 'float_kind' : sets 'float' and 'longfloat'
-            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
-            - 'str_kind' : sets 'str' and 'numpystr'
-
-    See Also
-    --------
-    get_printoptions, set_string_function, array2string
-
-    Notes
-    -----
-    `formatter` is always reset with a call to `set_printoptions`.
-
-    Examples
-    --------
-    Floating point precision can be set:
-
-    >>> np.set_printoptions(precision=4)
-    >>> print np.array([1.123456789])
-    [ 1.1235]
-
-    Long arrays can be summarised:
-
-    >>> np.set_printoptions(threshold=5)
-    >>> print np.arange(10)
-    [0 1 2 ..., 7 8 9]
-
-    Small results can be suppressed:
-
-    >>> eps = np.finfo(float).eps
-    >>> x = np.arange(4.)
-    >>> x**2 - (x + eps)**2
-    array([ -4.9304e-32,  -4.4409e-16,   0.0000e+00,   0.0000e+00])
-    >>> np.set_printoptions(suppress=True)
-    >>> x**2 - (x + eps)**2
-    array([-0., -0.,  0.,  0.])
-
-    A custom formatter can be used to display array elements as desired:
-
-    >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)})
-    >>> x = np.arange(3)
-    >>> x
-    array([int: 0, int: -1, int: -2])
-    >>> np.set_printoptions()  # formatter gets reset
-    >>> x
-    array([0, 1, 2])
-
-    To put back the default options, you can use:
-
-    >>> np.set_printoptions(edgeitems=3,infstr='inf',
-    ... linewidth=75, nanstr='nan', precision=8,
-    ... suppress=False, threshold=1000, formatter=None)
-    """
-
-    global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \
-           _line_width, _float_output_suppress_small, _nan_str, _inf_str, \
-           _formatter
-    if linewidth is not None:
-        _line_width = linewidth
-    if threshold is not None:
-        _summaryThreshold = threshold
-    if edgeitems is not None:
-        _summaryEdgeItems = edgeitems
-    if precision is not None:
-        _float_output_precision = precision
-    if suppress is not None:
-        _float_output_suppress_small = not not suppress
-    if nanstr is not None:
-        _nan_str = nanstr
-    if infstr is not None:
-        _inf_str = infstr
-    _formatter = formatter
-
-def get_printoptions():
-    """
-    Return the current print options.
-
-    Returns
-    -------
-    print_opts : dict
-        Dictionary of current print options with keys
-
-          - precision : int
-          - threshold : int
-          - edgeitems : int
-          - linewidth : int
-          - suppress : bool
-          - nanstr : str
-          - infstr : str
-          - formatter : dict of callables
-
-        For a full description of these options, see `set_printoptions`.
-
-    See Also
-    --------
-    set_printoptions, set_string_function
-
-    """
-    d = dict(precision=_float_output_precision,
-             threshold=_summaryThreshold,
-             edgeitems=_summaryEdgeItems,
-             linewidth=_line_width,
-             suppress=_float_output_suppress_small,
-             nanstr=_nan_str,
-             infstr=_inf_str,
-             formatter=_formatter)
-    return d
-
-def _leading_trailing(a):
-    import numeric as _nc
-    if a.ndim == 1:
-        if len(a) > 2*_summaryEdgeItems:
-            b = _nc.concatenate((a[:_summaryEdgeItems],
-                                     a[-_summaryEdgeItems:]))
-        else:
-            b = a
-    else:
-        if len(a) > 2*_summaryEdgeItems:
-            l = [_leading_trailing(a[i]) for i in range(
-                min(len(a), _summaryEdgeItems))]
-            l.extend([_leading_trailing(a[-i]) for i in range(
-                min(len(a), _summaryEdgeItems),0,-1)])
-        else:
-            l = [_leading_trailing(a[i]) for i in range(0, len(a))]
-        b = _nc.concatenate(tuple(l))
-    return b
-
-def _boolFormatter(x):
-    if x:
-        return ' True'
-    else:
-        return 'False'
-
-
-def repr_format(x):
-    return repr(x)
-
-def _array2string(a, max_line_width, precision, suppress_small, separator=' ',
-                  prefix="", formatter=None):
-
-    if max_line_width is None:
-        max_line_width = _line_width
-
-    if precision is None:
-        precision = _float_output_precision
-
-    if suppress_small is None:
-        suppress_small = _float_output_suppress_small
-
-    if formatter is None:
-        formatter = _formatter
-
-    if a.size > _summaryThreshold:
-        summary_insert = "..., "
-        data = _leading_trailing(a)
-    else:
-        summary_insert = ""
-        data = ravel(a)
-
-    formatdict = {'bool' : _boolFormatter,
-                  'int' : IntegerFormat(data),
-                  'float' : FloatFormat(data, precision, suppress_small),
-                  'longfloat' : FloatFormat(data, precision, suppress_small),
-                  'complexfloat' : ComplexFormat(data, precision,
-                                                 suppress_small),
-                  'longcomplexfloat' : ComplexFormat(data, precision,
-                                                     suppress_small),
-                  'datetime' : DatetimeFormat(data),
-                  'timedelta' : TimedeltaFormat(data),
-                  'numpystr' : repr_format,
-                  'str' : str}
-
-    if formatter is not None:
-        fkeys = [k for k in formatter.keys() if formatter[k] is not None]
-        if 'all' in fkeys:
-            for key in formatdict.keys():
-                formatdict[key] = formatter['all']
-        if 'int_kind' in fkeys:
-            for key in ['int']:
-                formatdict[key] = formatter['int_kind']
-        if 'float_kind' in fkeys:
-            for key in ['float', 'longfloat']:
-                formatdict[key] = formatter['float_kind']
-        if 'complex_kind' in fkeys:
-            for key in ['complexfloat', 'longcomplexfloat']:
-                formatdict[key] = formatter['complex_kind']
-        if 'str_kind' in fkeys:
-            for key in ['numpystr', 'str']:
-                formatdict[key] = formatter['str_kind']
-        for key in formatdict.keys():
-            if key in fkeys:
-                formatdict[key] = formatter[key]
-
-    try:
-        format_function = a._format
-        msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \
-              "will be removed in 2.1. Use the `formatter` kw instead."
-        import warnings
-        warnings.warn(msg, DeprecationWarning)
-    except AttributeError:
-        # find the right formatting function for the array
-        dtypeobj = a.dtype.type
-        if issubclass(dtypeobj, _nt.bool_):
-            format_function = formatdict['bool']
-        elif issubclass(dtypeobj, _nt.integer):
-            #if issubclass(dtypeobj, _nt.timedelta64):
-            #    format_function = formatdict['timedelta']
-            #else:
-            format_function = formatdict['int']
-        elif issubclass(dtypeobj, _nt.floating):
-            if hasattr(_nt, 'longfloat') and issubclass(dtypeobj, _nt.longfloat):
-                format_function = formatdict['longfloat']
-            else:
-                format_function = formatdict['float']
-        elif issubclass(dtypeobj, _nt.complexfloating):
-            if hasattr(_nt, 'clongfloat') and issubclass(dtypeobj, _nt.clongfloat):
-                format_function = formatdict['longcomplexfloat']
-            else:
-                format_function = formatdict['complexfloat']
-        elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)):
-            format_function = formatdict['numpystr']
-        #elif issubclass(dtypeobj, _nt.datetime64):
-        #    format_function = formatdict['datetime']
-        else:
-            format_function = formatdict['str']
-
-    # skip over "["
-    next_line_prefix = " "
-    # skip over array(
-    next_line_prefix += " "*len(prefix)
-
-    lst = _formatArray(a, format_function, len(a.shape), max_line_width,
-                       next_line_prefix, separator,
-                       _summaryEdgeItems, summary_insert)[:-1]
-    return lst
-
-def _convert_arrays(obj):
-    import numeric as _nc
-    newtup = []
-    for k in obj:
-        if isinstance(k, _nc.ndarray):
-            k = k.tolist()
-        elif isinstance(k, tuple):
-            k = _convert_arrays(k)
-        newtup.append(k)
-    return tuple(newtup)
-
-
-def array2string(a, max_line_width=None, precision=None,
-                 suppress_small=None, separator=' ', prefix="",
-                 style=repr, formatter=None):
-    """
-    Return a string representation of an array.
-
-    Parameters
-    ----------
-    a : ndarray
-        Input array.
-    max_line_width : int, optional
-        The maximum number of columns the string should span. Newline
-        characters splits the string appropriately after array elements.
-    precision : int, optional
-        Floating point precision. Default is the current printing
-        precision (usually 8), which can be altered using `set_printoptions`.
-    suppress_small : bool, optional
-        Represent very small numbers as zero. A number is "very small" if it
-        is smaller than the current printing precision.
-    separator : str, optional
-        Inserted between elements.
-    prefix : str, optional
-        An array is typically printed as::
-
-          'prefix(' + array2string(a) + ')'
-
-        The length of the prefix string is used to align the
-        output correctly.
-    style : function, optional
-        A function that accepts an ndarray and returns a string.  Used only
-        when the shape of `a` is equal to ``()``, i.e. for 0-D arrays.
-    formatter : dict of callables, optional
-        If not None, the keys should indicate the type(s) that the respective
-        formatting function applies to.  Callables should return a string.
-        Types that are not specified (by their corresponding keys) are handled
-        by the default formatters.  Individual types for which a formatter
-        can be set are::
-
-            - 'bool'
-            - 'int'
-            - 'timedelta' : a `numpy.timedelta64`
-            - 'datetime' : a `numpy.datetime64`
-            - 'float'
-            - 'longfloat' : 128-bit floats
-            - 'complexfloat'
-            - 'longcomplexfloat' : composed of two 128-bit floats
-            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
-            - 'str' : all other strings
-
-        Other keys that can be used to set a group of types at once are::
-
-            - 'all' : sets all types
-            - 'int_kind' : sets 'int'
-            - 'float_kind' : sets 'float' and 'longfloat'
-            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
-            - 'str_kind' : sets 'str' and 'numpystr'
-
-    Returns
-    -------
-    array_str : str
-        String representation of the array.
-
-    Raises
-    ------
-    TypeError : if a callable in `formatter` does not return a string.
-
-    See Also
-    --------
-    array_str, array_repr, set_printoptions, get_printoptions
-
-    Notes
-    -----
-    If a formatter is specified for a certain type, the `precision` keyword is
-    ignored for that type.
-
-    Examples
-    --------
-    >>> x = np.array([1e-16,1,2,3])
-    >>> print np.array2string(x, precision=2, separator=',',
-    ...                       suppress_small=True)
-    [ 0., 1., 2., 3.]
-
-    >>> x  = np.arange(3.)
-    >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x})
-    '[0.00 1.00 2.00]'
-
-    >>> x  = np.arange(3)
-    >>> np.array2string(x, formatter={'int':lambda x: hex(x)})
-    '[0x0L 0x1L 0x2L]'
-
-    """
-
-    if a.shape == ():
-        x = a.item()
-        try:
-            lst = a._format(x)
-            msg = "The `_format` attribute is deprecated in Numpy " \
-                  "2.0 and will be removed in 2.1. Use the " \
-                  "`formatter` kw instead."
-            import warnings
-            warnings.warn(msg, DeprecationWarning)
-        except AttributeError:
-            if isinstance(x, tuple):
-                x = _convert_arrays(x)
-            lst = style(x)
-    elif reduce(product, a.shape) == 0:
-        # treat as a null array if any of shape elements == 0
-        lst = "[]"
-    else:
-        lst = _array2string(a, max_line_width, precision, suppress_small,
-                            separator, prefix, formatter=formatter)
-    return lst
-
-def _extendLine(s, line, word, max_line_len, next_line_prefix):
-    if len(line.rstrip()) + len(word.rstrip()) >= max_line_len:
-        s += line.rstrip() + "\n"
-        line = next_line_prefix
-    line += word
-    return s, line
-
-
-def _formatArray(a, format_function, rank, max_line_len,
-                 next_line_prefix, separator, edge_items, summary_insert):
-    """formatArray is designed for two modes of operation:
-
-    1. Full output
-
-    2. Summarized output
-
-    """
-    if rank == 0:
-        obj = a.item()
-        if isinstance(obj, tuple):
-            obj = _convert_arrays(obj)
-        return str(obj)
-
-    if summary_insert and 2*edge_items < len(a):
-        leading_items, trailing_items, summary_insert1 = \
-                       edge_items, edge_items, summary_insert
-    else:
-        leading_items, trailing_items, summary_insert1 = 0, len(a), ""
-
-    if rank == 1:
-        s = ""
-        line = next_line_prefix
-        for i in xrange(leading_items):
-            word = format_function(a[i]) + separator
-            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-
-        if summary_insert1:
-            s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix)
-
-        for i in xrange(trailing_items, 1, -1):
-            word = format_function(a[-i]) + separator
-            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-
-        word = format_function(a[-1])
-        s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-        s += line + "]\n"
-        s = '[' + s[len(next_line_prefix):]
-    else:
-        s = '['
-        sep = separator.rstrip()
-        for i in xrange(leading_items):
-            if i > 0:
-                s += next_line_prefix
-            s += _formatArray(a[i], format_function, rank-1, max_line_len,
-                              " " + next_line_prefix, separator, edge_items,
-                              summary_insert)
-            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
-
-        if summary_insert1:
-            s += next_line_prefix + summary_insert1 + "\n"
-
-        for i in xrange(trailing_items, 1, -1):
-            if leading_items or i != trailing_items:
-                s += next_line_prefix
-            s += _formatArray(a[-i], format_function, rank-1, max_line_len,
-                              " " + next_line_prefix, separator, edge_items,
-                              summary_insert)
-            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
-        if leading_items or trailing_items > 1:
-            s += next_line_prefix
-        s += _formatArray(a[-1], format_function, rank-1, max_line_len,
-                          " " + next_line_prefix, separator, edge_items,
-                          summary_insert).rstrip()+']\n'
-    return s
-
-class FloatFormat(object):
-    def __init__(self, data, precision, suppress_small, sign=False):
-        self.precision = precision
-        self.suppress_small = suppress_small
-        self.sign = sign
-        self.exp_format = False
-        self.large_exponent = False
-        self.max_str_len = 0
-        try:
-            self.fillFormat(data)
-        except (TypeError, NotImplementedError):
-            # if reduce(data) fails, this instance will not be called, just
-            # instantiated in formatdict.
-            pass
-
-    def fillFormat(self, data):
-        import numeric as _nc
-        errstate = _nc.seterr(all='ignore')
-        try:
-            special = isnan(data) | isinf(data)
-            valid = not_equal(data, 0) & ~special
-            non_zero = absolute(data.compress(valid))
-            if len(non_zero) == 0:
-                max_val = 0.
-                min_val = 0.
-            else:
-                max_val = maximum.reduce(non_zero)
-                min_val = minimum.reduce(non_zero)
-                if max_val >= 1.e8:
-                    self.exp_format = True
-                if not self.suppress_small and (min_val < 0.0001
-                                           or max_val/min_val > 1000.):
-                    self.exp_format = True
-        finally:
-            _nc.seterr(**errstate)
-
-        if self.exp_format:
-            self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100
-            self.max_str_len = 8 + self.precision
-            if self.large_exponent:
-                self.max_str_len += 1
-            if self.sign:
-                format = '%+'
-            else:
-                format = '%'
-            format = format + '%d.%de' % (self.max_str_len, self.precision)
-        else:
-            format = '%%.%df' % (self.precision,)
-            if len(non_zero):
-                precision = max([_digits(x, self.precision, format)
-                                 for x in non_zero])
-            else:
-                precision = 0
-            precision = min(self.precision, precision)
-            self.max_str_len = len(str(int(max_val))) + precision + 2
-            if _nc.any(special):
-                self.max_str_len = max(self.max_str_len,
-                                       len(_nan_str),
-                                       len(_inf_str)+1)
-            if self.sign:
-                format = '%#+'
-            else:
-                format = '%#'
-            format = format + '%d.%df' % (self.max_str_len, precision)
-
-        self.special_fmt = '%%%ds' % (self.max_str_len,)
-        self.format = format
-
-    def __call__(self, x, strip_zeros=True):
-        import numeric as _nc
-        err = _nc.seterr(invalid='ignore')
-        try:
-            if isnan(x):
-                if self.sign:
-                    return self.special_fmt % ('+' + _nan_str,)
-                else:
-                    return self.special_fmt % (_nan_str,)
-            elif isinf(x):
-                if x > 0:
-                    if self.sign:
-                        return self.special_fmt % ('+' + _inf_str,)
-                    else:
-                        return self.special_fmt % (_inf_str,)
-                else:
-                    return self.special_fmt % ('-' + _inf_str,)
-        finally:
-            _nc.seterr(**err)
-
-        s = self.format % x
-        if self.large_exponent:
-            # 3-digit exponent
-            expsign = s[-3]
-            if expsign == '+' or expsign == '-':
-                s = s[1:-2] + '0' + s[-2:]
-        elif self.exp_format:
-            # 2-digit exponent
-            if s[-3] == '0':
-                s = ' ' + s[:-3] + s[-2:]
-        elif strip_zeros:
-            z = s.rstrip('0')
-            s = z + ' '*(len(s)-len(z))
-        return s
-
-
-def _digits(x, precision, format):
-    s = format % x
-    z = s.rstrip('0')
-    return precision - len(s) + len(z)
-
-
-_MAXINT = sys.maxint
-_MININT = -sys.maxint-1
-class IntegerFormat(object):
-    def __init__(self, data):
-        try:
-            max_str_len = max(len(str(maximum.reduce(data))),
-                              len(str(minimum.reduce(data))))
-            self.format = '%' + str(max_str_len) + 'd'
-        except (TypeError, NotImplementedError):
-            # if reduce(data) fails, this instance will not be called, just
-            # instantiated in formatdict.
-            pass
-        except ValueError:
-            # this occurs when everything is NA
-            pass
-
-    def __call__(self, x):
-        if _MININT < x < _MAXINT:
-            return self.format % x
-        else:
-            return "%s" % x
-
-class LongFloatFormat(object):
-    # XXX Have to add something to determine the width to use a la FloatFormat
-    # Right now, things won't line up properly
-    def __init__(self, precision, sign=False):
-        self.precision = precision
-        self.sign = sign
-
-    def __call__(self, x):
-        if isnan(x):
-            if self.sign:
-                return '+' + _nan_str
-            else:
-                return ' ' + _nan_str
-        elif isinf(x):
-            if x > 0:
-                if self.sign:
-                    return '+' + _inf_str
-                else:
-                    return ' ' + _inf_str
-            else:
-                return '-' + _inf_str
-        elif x >= 0:
-            if self.sign:
-                return '+' + format_longfloat(x, self.precision)
-            else:
-                return ' ' + format_longfloat(x, self.precision)
-        else:
-            return format_longfloat(x, self.precision)
-
-
-class LongComplexFormat(object):
-    def __init__(self, precision):
-        self.real_format = LongFloatFormat(precision)
-        self.imag_format = LongFloatFormat(precision, sign=True)
-
-    def __call__(self, x):
-        r = self.real_format(x.real)
-        i = self.imag_format(x.imag)
-        return r + i + 'j'
-
-
-class ComplexFormat(object):
-    def __init__(self, x, precision, suppress_small):
-        self.real_format = FloatFormat(x.real, precision, suppress_small)
-        self.imag_format = FloatFormat(x.imag, precision, suppress_small,
-                                       sign=True)
-
-    def __call__(self, x):
-        r = self.real_format(x.real, strip_zeros=False)
-        i = self.imag_format(x.imag, strip_zeros=False)
-        if not self.imag_format.exp_format:
-            z = i.rstrip('0')
-            i = z + 'j' + ' '*(len(i)-len(z))
-        else:
-            i = i + 'j'
-        return r + i
-
-class DatetimeFormat(object):
-    def __init__(self, x, unit=None,
-                timezone=None, casting='same_kind'):
-        # Get the unit from the dtype
-        if unit is None:
-            if x.dtype.kind == 'M':
-                unit = datetime_data(x.dtype)[0]
-            else:
-                unit = 's'
-
-        # If timezone is default, make it 'local' or 'UTC' based on the unit
-        if timezone is None:
-            # Date units -> UTC, time units -> local
-            if unit in ('Y', 'M', 'W', 'D'):
-                self.timezone = 'UTC'
-            else:
-                self.timezone = 'local'
-        else:
-            self.timezone = timezone
-        self.unit = unit
-        self.casting = casting
-
-    def __call__(self, x):
-        return "'%s'" % datetime_as_string(x,
-                                    unit=self.unit,
-                                    timezone=self.timezone,
-                                    casting=self.casting)
-
-class TimedeltaFormat(object):
-    def __init__(self, data):
-        if data.dtype.kind == 'm':
-            v = data.view('i8')
-            max_str_len = max(len(str(maximum.reduce(v))),
-                              len(str(minimum.reduce(v))))
-            self.format = '%' + str(max_str_len) + 'd'
-
-    def __call__(self, x):
-        return self.format % x.astype('i8')
-
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ /dev/null
@@ -1,2924 +0,0 @@
-######################################################################
-# This is a copy of numpy/core/fromnumeric.py modified for numpypy
-######################################################################
-"""Module containing non-deprecated functions borrowed from Numeric.
-
-"""
-from __future__ import division, absolute_import, print_function
-
-import types
-
-from . import multiarray as mu
-from . import umath as um
-from . import numerictypes as nt
-from .numeric import asarray, array, asanyarray, concatenate
-from . import _methods
-
-
-# functions that are methods
-__all__ = [
-        'alen', 'all', 'alltrue', 'amax', 'amin', 'any', 'argmax',
-        'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip',
-        'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean',
-        'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put',
-        'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_',
-        'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze',
-        'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var',
-        ]
-
-
-try:
-    _gentype = types.GeneratorType
-except AttributeError:
-    _gentype = type(None)
-
-# save away Python sum
-_sum_ = sum
-
-# functions that are now methods
-def _wrapit(obj, method, *args, **kwds):
-    try:
-        wrap = obj.__array_wrap__
-    except AttributeError:
-        wrap = None
-    result = getattr(asarray(obj), method)(*args, **kwds)
-    if wrap:
-        if not isinstance(result, mu.ndarray):
-            result = asarray(result)
-        result = wrap(result)
-    return result
-
-
-def take(a, indices, axis=None, out=None, mode='raise'):
-    """
-    Take elements from an array along an axis.
-
-    This function does the same thing as "fancy" indexing (indexing arrays
-    using arrays); however, it can be easier to use if you need elements
-    along a given axis.
-
-    Parameters
-    ----------
-    a : array_like
-        The source array.
-    indices : array_like
-        The indices of the values to extract.
-
-        .. versionadded:: 1.8.0
-
-        Also allow scalars for indices.
-    axis : int, optional
-        The axis over which to select values. By default, the flattened
-        input array is used.
-    out : ndarray, optional
-        If provided, the result will be placed in this array. It should
-        be of the appropriate shape and dtype.
-    mode : {'raise', 'wrap', 'clip'}, optional
-        Specifies how out-of-bounds indices will behave.
-
-        * 'raise' -- raise an error (default)
-        * 'wrap' -- wrap around
-        * 'clip' -- clip to the range
-
-        'clip' mode means that all indices that are too large are replaced
-        by the index that addresses the last element along that axis. Note
-        that this disables indexing with negative numbers.
-
-    Returns
-    -------
-    subarray : ndarray
-        The returned array has the same type as `a`.
-
-    See Also
-    --------
-    ndarray.take : equivalent method
-
-    Examples
-    --------
-    >>> a = [4, 3, 5, 7, 6, 8]
-    >>> indices = [0, 1, 4]
-    >>> np.take(a, indices)
-    array([4, 3, 6])
-
-    In this example if `a` is an ndarray, "fancy" indexing can be used.
-
-    >>> a = np.array(a)
-    >>> a[indices]
-    array([4, 3, 6])
-
-    If `indices` is not one dimensional, the output also has these dimensions.
-
-    >>> np.take(a, [[0, 1], [2, 3]])
-    array([[4, 3],
-           [5, 7]])
-    """
-    try:
-        take = a.take
-    except AttributeError:
-        return _wrapit(a, 'take', indices, axis, out, mode)
-    return take(indices, axis, out, mode)
-
-
-# not deprecated --- copy if necessary, view otherwise
-def reshape(a, newshape, order='C'):
-    """
-    Gives a new shape to an array without changing its data.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be reshaped.
-    newshape : int or tuple of ints
-        The new shape should be compatible with the original shape. If
-        an integer, then the result will be a 1-D array of that length.
-        One shape dimension can be -1. In this case, the value is inferred
-        from the length of the array and remaining dimensions.
-    order : {'C', 'F', 'A'}, optional
-        Read the elements of `a` using this index order, and place the elements
-        into the reshaped array using this index order.  'C' means to
-        read / write the elements using C-like index order, with the last axis index
-        changing fastest, back to the first axis index changing slowest.  'F'
-        means to read / write the elements using Fortran-like index order, with
-        the first index changing fastest, and the last index changing slowest.
-        Note that the 'C' and 'F' options take no account of the memory layout
-        of the underlying array, and only refer to the order of indexing.  'A'
-        means to read / write the elements in Fortran-like index order if `a` is
-        Fortran *contiguous* in memory, C-like order otherwise.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        This will be a new view object if possible; otherwise, it will
-        be a copy.  Note there is no guarantee of the *memory layout* (C- or
-        Fortran- contiguous) of the returned array.
-
-    See Also
-    --------
-    ndarray.reshape : Equivalent method.
-
-    Notes
-    -----
-    It is not always possible to change the shape of an array without
-    copying the data. If you want an error to be raise if the data is copied,
-    you should assign the new shape to the shape attribute of the array::
-
-     >>> a = np.zeros((10, 2))
-     # A transpose make the array non-contiguous
-     >>> b = a.T
-     # Taking a view makes it possible to modify the shape without modifying the
-     # initial object.
-     >>> c = b.view()
-     >>> c.shape = (20)
-     AttributeError: incompatible shape for a non-contiguous array
-
-    The `order` keyword gives the index ordering both for *fetching* the values
-    from `a`, and then *placing* the values into the output array.  For example,
-    let's say you have an array:
-
-    >>> a = np.arange(6).reshape((3, 2))
-    >>> a
-    array([[0, 1],
-           [2, 3],
-           [4, 5]])
-
-    You can think of reshaping as first raveling the array (using the given
-    index order), then inserting the elements from the raveled array into the
-    new array using the same kind of index ordering as was used for the
-    raveling.
-
-    >>> np.reshape(a, (2, 3)) # C-like index ordering
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.reshape(a, (2, 3), order='F') # Fortran-like index ordering
-    array([[0, 4, 3],
-           [2, 1, 5]])
-    >>> np.reshape(np.ravel(a, order='F'), (2, 3), order='F')
-    array([[0, 4, 3],
-           [2, 1, 5]])
-
-    Examples
-    --------
-    >>> a = np.array([[1,2,3], [4,5,6]])
-    >>> np.reshape(a, 6)
-    array([1, 2, 3, 4, 5, 6])
-    >>> np.reshape(a, 6, order='F')
-    array([1, 4, 2, 5, 3, 6])
-
-    >>> np.reshape(a, (3,-1))       # the unspecified value is inferred to be 2
-    array([[1, 2],
-           [3, 4],
-           [5, 6]])
-    """
-    assert order == 'C'
-    try:
-        reshape = a.reshape
-    except AttributeError:
-        return _wrapit(a, 'reshape', newshape)
-    return reshape(newshape)
-
-
-def choose(a, choices, out=None, mode='raise'):
-    """
-    Construct an array from an index array and a set of arrays to choose from.
-
-    First of all, if confused or uncertain, definitely look at the Examples -
-    in its full generality, this function is less simple than it might
-    seem from the following code description (below ndi =
-    `numpy.lib.index_tricks`):
-
-    ``np.choose(a,c) == np.array([c[a[I]][I] for I in ndi.ndindex(a.shape)])``.
-
-    But this omits some subtleties.  Here is a fully general summary:
-
-    Given an "index" array (`a`) of integers and a sequence of `n` arrays
-    (`choices`), `a` and each choice array are first broadcast, as necessary,
-    to arrays of a common shape; calling these *Ba* and *Bchoices[i], i =
-    0,...,n-1* we have that, necessarily, ``Ba.shape == Bchoices[i].shape``
-    for each `i`.  Then, a new array with shape ``Ba.shape`` is created as
-    follows:
-
-    * if ``mode=raise`` (the default), then, first of all, each element of
-      `a` (and thus `Ba`) must be in the range `[0, n-1]`; now, suppose that
-      `i` (in that range) is the value at the `(j0, j1, ..., jm)` position
-      in `Ba` - then the value at the same position in the new array is the
-      value in `Bchoices[i]` at that same position;
-
-    * if ``mode=wrap``, values in `a` (and thus `Ba`) may be any (signed)
-      integer; modular arithmetic is used to map integers outside the range
-      `[0, n-1]` back into that range; and then the new array is constructed
-      as above;
-
-    * if ``mode=clip``, values in `a` (and thus `Ba`) may be any (signed)
-      integer; negative integers are mapped to 0; values greater than `n-1`
-      are mapped to `n-1`; and then the new array is constructed as above.
-
-    Parameters
-    ----------
-    a : int array
-        This array must contain integers in `[0, n-1]`, where `n` is the number
-        of choices, unless ``mode=wrap`` or ``mode=clip``, in which cases any
-        integers are permissible.
-    choices : sequence of arrays
-        Choice arrays. `a` and all of the choices must be broadcastable to the
-        same shape.  If `choices` is itself an array (not recommended), then
-        its outermost dimension (i.e., the one corresponding to
-        ``choices.shape[0]``) is taken as defining the "sequence".
-    out : array, optional
-        If provided, the result will be inserted into this array. It should
-        be of the appropriate shape and dtype.
-    mode : {'raise' (default), 'wrap', 'clip'}, optional
-        Specifies how indices outside `[0, n-1]` will be treated:
-
-          * 'raise' : an exception is raised
-          * 'wrap' : value becomes value mod `n`
-          * 'clip' : values < 0 are mapped to 0, values > n-1 are mapped to n-1
-
-    Returns
-    -------
-    merged_array : array
-        The merged result.
-
-    Raises
-    ------
-    ValueError: shape mismatch
-        If `a` and each choice array are not all broadcastable to the same
-        shape.
-
-    See Also
-    --------
-    ndarray.choose : equivalent method
-
-    Notes
-    -----
-    To reduce the chance of misinterpretation, even though the following
-    "abuse" is nominally supported, `choices` should neither be, nor be
-    thought of as, a single array, i.e., the outermost sequence-like container
-    should be either a list or a tuple.
-
-    Examples
-    --------
-
-    >>> choices = [[0, 1, 2, 3], [10, 11, 12, 13],
-    ...   [20, 21, 22, 23], [30, 31, 32, 33]]
-    >>> np.choose([2, 3, 1, 0], choices
-    ... # the first element of the result will be the first element of the
-    ... # third (2+1) "array" in choices, namely, 20; the second element
-    ... # will be the second element of the fourth (3+1) choice array, i.e.,
-    ... # 31, etc.
-    ... )
-    array([20, 31, 12,  3])
-    >>> np.choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1)
-    array([20, 31, 12,  3])
-    >>> # because there are 4 choice arrays
-    >>> np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4)
-    array([20,  1, 12,  3])
-    >>> # i.e., 0
-
-    A couple examples illustrating how choose broadcasts:
-
-    >>> a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]]
-    >>> choices = [-10, 10]
-    >>> np.choose(a, choices)
-    array([[ 10, -10,  10],
-           [-10,  10, -10],
-           [ 10, -10,  10]])
-
-    >>> # With thanks to Anne Archibald
-    >>> a = np.array([0, 1]).reshape((2,1,1))
-    >>> c1 = np.array([1, 2, 3]).reshape((1,3,1))
-    >>> c2 = np.array([-1, -2, -3, -4, -5]).reshape((1,1,5))
-    >>> np.choose(a, (c1, c2)) # result is 2x3x5, res[0,:,:]=c1, res[1,:,:]=c2
-    array([[[ 1,  1,  1,  1,  1],
-            [ 2,  2,  2,  2,  2],
-            [ 3,  3,  3,  3,  3]],
-           [[-1, -2, -3, -4, -5],
-            [-1, -2, -3, -4, -5],
-            [-1, -2, -3, -4, -5]]])
-
-    """
-    try:
-        choose = a.choose
-    except AttributeError:
-        return _wrapit(a, 'choose', choices, out=out, mode=mode)
-    return choose(choices, out=out, mode=mode)
-
-
-def repeat(a, repeats, axis=None):
-    """
-    Repeat elements of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    repeats : {int, array of ints}
-        The number of repetitions for each element.  `repeats` is broadcasted
-        to fit the shape of the given axis.
-    axis : int, optional
-        The axis along which to repeat values.  By default, use the
-        flattened input array, and return a flat output array.
-
-    Returns
-    -------
-    repeated_array : ndarray
-        Output array which has the same shape as `a`, except along
-        the given axis.
-
-    See Also
-    --------
-    tile : Tile an array.
-
-    Examples
-    --------
-    >>> x = np.array([[1,2],[3,4]])
-    >>> np.repeat(x, 2)
-    array([1, 1, 2, 2, 3, 3, 4, 4])
-    >>> np.repeat(x, 3, axis=1)
-    array([[1, 1, 1, 2, 2, 2],
-           [3, 3, 3, 4, 4, 4]])
-    >>> np.repeat(x, [1, 2], axis=0)
-    array([[1, 2],
-           [3, 4],
-           [3, 4]])
-
-    """
-    try:
-        repeat = a.repeat
-    except AttributeError:
-        return _wrapit(a, 'repeat', repeats, axis)
-    return repeat(repeats, axis)
-
-
-def put(a, ind, v, mode='raise'):
-    """
-    Replaces specified elements of an array with given values.
-
-    The indexing works on the flattened target array. `put` is roughly
-    equivalent to:
-
-    ::
-
-        a.flat[ind] = v
-
-    Parameters
-    ----------
-    a : ndarray
-        Target array.
-    ind : array_like
-        Target indices, interpreted as integers.
-    v : array_like
-        Values to place in `a` at target indices. If `v` is shorter than
-        `ind` it will be repeated as necessary.
-    mode : {'raise', 'wrap', 'clip'}, optional
-        Specifies how out-of-bounds indices will behave.
-
-        * 'raise' -- raise an error (default)
-        * 'wrap' -- wrap around
-        * 'clip' -- clip to the range
-
-        'clip' mode means that all indices that are too large are replaced
-        by the index that addresses the last element along that axis. Note
-        that this disables indexing with negative numbers.
-
-    See Also
-    --------
-    putmask, place
-
-    Examples
-    --------
-    >>> a = np.arange(5)
-    >>> np.put(a, [0, 2], [-44, -55])
-    >>> a
-    array([-44,   1, -55,   3,   4])
-
-    >>> a = np.arange(5)
-    >>> np.put(a, 22, -5, mode='clip')
-    >>> a
-    array([ 0,  1,  2,  3, -5])
-
-    """
-    return a.put(ind, v, mode)
-
-
-def swapaxes(a, axis1, axis2):
-    """
-    Interchange two axes of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axis1 : int
-        First axis.
-    axis2 : int
-        Second axis.
-
-    Returns
-    -------
-    a_swapped : ndarray
-        If `a` is an ndarray, then a view of `a` is returned; otherwise
-        a new array is created.
-
-    Examples
-    --------
-    >>> x = np.array([[1,2,3]])
-    >>> np.swapaxes(x,0,1)
-    array([[1],
-           [2],
-           [3]])
-
-    >>> x = np.array([[[0,1],[2,3]],[[4,5],[6,7]]])
-    >>> x
-    array([[[0, 1],
-            [2, 3]],
-           [[4, 5],
-            [6, 7]]])
-
-    >>> np.swapaxes(x,0,2)
-    array([[[0, 4],
-            [2, 6]],
-           [[1, 5],
-            [3, 7]]])
-
-    """
-    try:
-        swapaxes = a.swapaxes
-    except AttributeError:
-        return _wrapit(a, 'swapaxes', axis1, axis2)
-    return swapaxes(axis1, axis2)
-
-
-def transpose(a, axes=None):
-    """
-    Permute the dimensions of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axes : list of ints, optional
-        By default, reverse the dimensions, otherwise permute the axes
-        according to the values given.
-
-    Returns
-    -------
-    p : ndarray
-        `a` with its axes permuted.  A view is returned whenever
-        possible.
-
-    See Also
-    --------
-    rollaxis
-
-    Examples
-    --------
-    >>> x = np.arange(4).reshape((2,2))
-    >>> x
-    array([[0, 1],
-           [2, 3]])
-
-    >>> np.transpose(x)
-    array([[0, 2],
-           [1, 3]])
-
-    >>> x = np.ones((1, 2, 3))
-    >>> np.transpose(x, (1, 0, 2)).shape
-    (2, 1, 3)
-
-    """
-    if axes is not None:
-        raise NotImplementedError('No "axes" arg yet.')
-    try:
-        transpose = a.transpose
-    except AttributeError:
-        return _wrapit(a, 'transpose')
-    return transpose()
-
-
-def partition(a, kth, axis=-1, kind='introselect', order=None):
-    """
-    Return a partitioned copy of an array.
-
-    Creates a copy of the array with its elements rearranged in such a way that
-    the value of the element in kth position is in the position it would be in
-    a sorted array. All elements smaller than the kth element are moved before
-    this element and all equal or greater are moved behind it. The ordering of
-    the elements in the two partitions is undefined.
-
-    .. versionadded:: 1.8.0
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be sorted.
-    kth : int or sequence of ints
-        Element index to partition by. The kth value of the element will be in
-        its final sorted position and all smaller elements will be moved before
-        it and all equal or greater elements behind it.
-        The order all elements in the partitions is undefined.
-        If provided with a sequence of kth it will partition all elements
-        indexed by kth  of them into their sorted position at once.
-    axis : int or None, optional
-        Axis along which to sort. If None, the array is flattened before
-        sorting. The default is -1, which sorts along the last axis.
-    kind : {'introselect'}, optional
-        Selection algorithm. Default is 'introselect'.
-    order : list, optional
-        When `a` is a structured array, this argument specifies which fields
-        to compare first, second, and so on.  This list does not need to
-        include all of the fields.
-
-    Returns
-    -------
-    partitioned_array : ndarray
-        Array of the same type and shape as `a`.
-
-    See Also
-    --------
-    ndarray.partition : Method to sort an array in-place.
-    argpartition : Indirect partition.
-    sort : Full sorting
-
-    Notes
-    -----
-    The various selection algorithms are characterized by their average speed,
-    worst case performance, work space size, and whether they are stable. A
-    stable sort keeps items with the same key in the same relative order. The
-    three available algorithms have the following properties:
-
-    ================= ======= ============= ============ =======
-       kind            speed   worst case    work space  stable
-    ================= ======= ============= ============ =======
-    'introselect'        1        O(n)           0         no
-    ================= ======= ============= ============ =======
-
-    All the partition algorithms make temporary copies of the data when
-    partitioning along any but the last axis.  Consequently, partitioning
-    along the last axis is faster and uses less space than partitioning
-    along any other axis.
-
-    The sort order for complex numbers is lexicographic. If both the real
-    and imaginary parts are non-nan then the order is determined by the
-    real parts except when they are equal, in which case the order is
-    determined by the imaginary parts.
-
-    Examples
-    --------
-    >>> a = np.array([3, 4, 2, 1])
-    >>> np.partition(a, 3)
-    array([2, 1, 3, 4])
-
-    >>> np.partition(a, (1, 3))
-    array([1, 2, 3, 4])
-
-    """
-    if axis is None:
-        a = asanyarray(a).flatten()
-        axis = 0
-    else:
-        a = asanyarray(a).copy()
-    a.partition(kth, axis=axis, kind=kind, order=order)
-    return a
-
-
-def argpartition(a, kth, axis=-1, kind='introselect', order=None):
-    """
-    Perform an indirect partition along the given axis using the algorithm
-    specified by the `kind` keyword. It returns an array of indices of the
-    same shape as `a` that index data along the given axis in partitioned
-    order.
-
-    .. versionadded:: 1.8.0
-
-    Parameters
-    ----------
-    a : array_like
-        Array to sort.
-    kth : int or sequence of ints
-        Element index to partition by. The kth element will be in its final
-        sorted position and all smaller elements will be moved before it and
-        all larger elements behind it.
-        The order all elements in the partitions is undefined.
-        If provided with a sequence of kth it will partition all of them into
-        their sorted position at once.
-    axis : int or None, optional
-        Axis along which to sort.  The default is -1 (the last axis). If None,
-        the flattened array is used.
-    kind : {'introselect'}, optional
-        Selection algorithm. Default is 'introselect'
-    order : list, optional
-        When `a` is an array with fields defined, this argument specifies
-        which fields to compare first, second, etc.  Not all fields need be
-        specified.
-
-    Returns
-    -------
-    index_array : ndarray, int
-        Array of indices that partition `a` along the specified axis.
-        In other words, ``a[index_array]`` yields a sorted `a`.
-
-    See Also
-    --------
-    partition : Describes partition algorithms used.
-    ndarray.partition : Inplace partition.
-    argsort : Full indirect sort
-
-    Notes
-    -----
-    See `partition` for notes on the different selection algorithms.
-
-    Examples
-    --------
-    One dimensional array:
-
-    >>> x = np.array([3, 4, 2, 1])
-    >>> x[np.argpartition(x, 3)]
-    array([2, 1, 3, 4])
-    >>> x[np.argpartition(x, (1, 3))]
-    array([1, 2, 3, 4])
-
-    """
-    return a.argpartition(kth, axis, kind=kind, order=order)
-
-
-def sort(a, axis=-1, kind='quicksort', order=None):
-    """
-    Return a sorted copy of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be sorted.
-    axis : int or None, optional
-        Axis along which to sort. If None, the array is flattened before
-        sorting. The default is -1, which sorts along the last axis.
-    kind : {'quicksort', 'mergesort', 'heapsort'}, optional
-        Sorting algorithm. Default is 'quicksort'.
-    order : list, optional
-        When `a` is a structured array, this argument specifies which fields
-        to compare first, second, and so on.  This list does not need to
-        include all of the fields.
-
-    Returns
-    -------
-    sorted_array : ndarray
-        Array of the same type and shape as `a`.
-
-    See Also
-    --------
-    ndarray.sort : Method to sort an array in-place.
-    argsort : Indirect sort.
-    lexsort : Indirect stable sort on multiple keys.
-    searchsorted : Find elements in a sorted array.
-    partition : Partial sort.
-
-    Notes
-    -----
-    The various sorting algorithms are characterized by their average speed,
-    worst case performance, work space size, and whether they are stable. A
-    stable sort keeps items with the same key in the same relative
-    order. The three available algorithms have the following
-    properties:
-
-    =========== ======= ============= ============ =======
-       kind      speed   worst case    work space  stable
-    =========== ======= ============= ============ =======
-    'quicksort'    1     O(n^2)            0          no
-    'mergesort'    2     O(n*log(n))      ~n/2        yes
-    'heapsort'     3     O(n*log(n))       0          no
-    =========== ======= ============= ============ =======
-
-    All the sort algorithms make temporary copies of the data when
-    sorting along any but the last axis.  Consequently, sorting along
-    the last axis is faster and uses less space than sorting along
-    any other axis.
-
-    The sort order for complex numbers is lexicographic. If both the real
-    and imaginary parts are non-nan then the order is determined by the
-    real parts except when they are equal, in which case the order is
-    determined by the imaginary parts.
-
-    Previous to numpy 1.4.0 sorting real and complex arrays containing nan
-    values led to undefined behaviour. In numpy versions >= 1.4.0 nan
-    values are sorted to the end. The extended sort order is:
-
-      * Real: [R, nan]
-      * Complex: [R + Rj, R + nanj, nan + Rj, nan + nanj]
-
-    where R is a non-nan real value. Complex values with the same nan
-    placements are sorted according to the non-nan part if it exists.
-    Non-nan values are sorted as before.
-
-    Examples
-    --------
-    >>> a = np.array([[1,4],[3,1]])
-    >>> np.sort(a)                # sort along the last axis
-    array([[1, 4],
-           [1, 3]])
-    >>> np.sort(a, axis=None)     # sort the flattened array
-    array([1, 1, 3, 4])
-    >>> np.sort(a, axis=0)        # sort along the first axis
-    array([[1, 1],
-           [3, 4]])
-
-    Use the `order` keyword to specify a field to use when sorting a
-    structured array:
-
-    >>> dtype = [('name', 'S10'), ('height', float), ('age', int)]
-    >>> values = [('Arthur', 1.8, 41), ('Lancelot', 1.9, 38),
-    ...           ('Galahad', 1.7, 38)]
-    >>> a = np.array(values, dtype=dtype)       # create a structured array
-    >>> np.sort(a, order='height')                        # doctest: +SKIP
-    array([('Galahad', 1.7, 38), ('Arthur', 1.8, 41),
-           ('Lancelot', 1.8999999999999999, 38)],
-          dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
-
-    Sort by age, then height if ages are equal:
-
-    >>> np.sort(a, order=['age', 'height'])               # doctest: +SKIP
-    array([('Galahad', 1.7, 38), ('Lancelot', 1.8999999999999999, 38),
-           ('Arthur', 1.8, 41)],
-          dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
-
-    """
-    if axis is None:
-        a = asanyarray(a).flatten()
-        axis = 0
-    else:
-        a = asanyarray(a).copy()
-    a.sort(axis, kind, order)
-    return a
-
-
-def argsort(a, axis=-1, kind='quicksort', order=None):
-    """
-    Returns the indices that would sort an array.
-
-    Perform an indirect sort along the given axis using the algorithm specified
-    by the `kind` keyword. It returns an array of indices of the same shape as
-    `a` that index data along the given axis in sorted order.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to sort.
-    axis : int or None, optional
-        Axis along which to sort.  The default is -1 (the last axis). If None,
-        the flattened array is used.
-    kind : {'quicksort', 'mergesort', 'heapsort'}, optional
-        Sorting algorithm.
-    order : list, optional
-        When `a` is an array with fields defined, this argument specifies
-        which fields to compare first, second, etc.  Not all fields need be
-        specified.
-
-    Returns
-    -------
-    index_array : ndarray, int
-        Array of indices that sort `a` along the specified axis.
-        In other words, ``a[index_array]`` yields a sorted `a`.
-
-    See Also
-    --------
-    sort : Describes sorting algorithms used.
-    lexsort : Indirect stable sort with multiple keys.
-    ndarray.sort : Inplace sort.
-    argpartition : Indirect partial sort.
-
-    Notes
-    -----
-    See `sort` for notes on the different sorting algorithms.
-
-    As of NumPy 1.4.0 `argsort` works with real/complex arrays containing
-    nan values. The enhanced sort order is documented in `sort`.
-
-    Examples
-    --------
-    One dimensional array:
-
-    >>> x = np.array([3, 1, 2])
-    >>> np.argsort(x)
-    array([1, 2, 0])
-
-    Two-dimensional array:
-
-    >>> x = np.array([[0, 3], [2, 2]])
-    >>> x
-    array([[0, 3],
-           [2, 2]])
-
-    >>> np.argsort(x, axis=0)
-    array([[0, 1],
-           [1, 0]])
-
-    >>> np.argsort(x, axis=1)
-    array([[0, 1],
-           [0, 1]])
-
-    Sorting with keys:
-
-    >>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '<i4'), ('y', '<i4')])
-    >>> x
-    array([(1, 0), (0, 1)],
-          dtype=[('x', '<i4'), ('y', '<i4')])
-
-    >>> np.argsort(x, order=('x','y'))
-    array([1, 0])
-
-    >>> np.argsort(x, order=('y','x'))
-    array([0, 1])
-
-    """
-    try:
-        argsort = a.argsort
-    except AttributeError:
-        return _wrapit(a, 'argsort', axis, kind, order)
-    return argsort(axis, kind, order)
-
-
-def argmax(a, axis=None):
-    """
-    Indices of the maximum values along an axis.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axis : int, optional
-        By default, the index is into the flattened array, otherwise
-        along the specified axis.
-
-    Returns
-    -------
-    index_array : ndarray of ints
-        Array of indices into the array. It has the same shape as `a.shape`
-        with the dimension along `axis` removed.
-
-    See Also
-    --------
-    ndarray.argmax, argmin
-    amax : The maximum value along a given axis.
-    unravel_index : Convert a flat index into an index tuple.
-
-    Notes
-    -----
-    In case of multiple occurrences of the maximum values, the indices
-    corresponding to the first occurrence are returned.
-
-    Examples
-    --------
-    >>> a = np.arange(6).reshape(2,3)
-    >>> a
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.argmax(a)
-    5
-    >>> np.argmax(a, axis=0)
-    array([1, 1, 1])
-    >>> np.argmax(a, axis=1)
-    array([2, 2])
-
-    >>> b = np.arange(6)
-    >>> b[1] = 5
-    >>> b
-    array([0, 5, 2, 3, 4, 5])
-    >>> np.argmax(b) # Only the first occurrence is returned.
-    1
-
-    """
-    assert axis is None
-    try:
-        argmax = a.argmax
-    except AttributeError:
-        return _wrapit(a, 'argmax')
-    return argmax()
-
-
-def argmin(a, axis=None):
-    """
-    Return the indices of the minimum values along an axis.
-
-    See Also
-    --------
-    argmax : Similar function.  Please refer to `numpy.argmax` for detailed
-        documentation.
-
-    """
-    assert axis is None
-    try:
-        argmin = a.argmin
-    except AttributeError:
-        return _wrapit(a, 'argmin')
-    return argmin()
-
-
-def searchsorted(a, v, side='left', sorter=None):
-    """
-    Find indices where elements should be inserted to maintain order.
-
-    Find the indices into a sorted array `a` such that, if the
-    corresponding elements in `v` were inserted before the indices, the
-    order of `a` would be preserved.
-
-    Parameters
-    ----------
-    a : 1-D array_like
-        Input array. If `sorter` is None, then it must be sorted in
-        ascending order, otherwise `sorter` must be an array of indices
-        that sort it.
-    v : array_like
-        Values to insert into `a`.
-    side : {'left', 'right'}, optional
-        If 'left', the index of the first suitable location found is given.
-        If 'right', return the last such index.  If there is no suitable
-        index, return either 0 or N (where N is the length of `a`).
-    sorter : 1-D array_like, optional
-        .. versionadded:: 1.7.0
-        Optional array of integer indices that sort array a into ascending
-        order. They are typically the result of argsort.
-
-    Returns
-    -------
-    indices : array of ints
-        Array of insertion points with the same shape as `v`.
-
-    See Also
-    --------
-    sort : Return a sorted copy of an array.
-    histogram : Produce histogram from 1-D data.
-
-    Notes
-    -----
-    Binary search is used to find the required insertion points.
-
-    As of Numpy 1.4.0 `searchsorted` works with real/complex arrays containing
-    `nan` values. The enhanced sort order is documented in `sort`.
-
-    Examples
-    --------
-    >>> np.searchsorted([1,2,3,4,5], 3)
-    2
-    >>> np.searchsorted([1,2,3,4,5], 3, side='right')
-    3
-    >>> np.searchsorted([1,2,3,4,5], [-10, 10, 2, 3])
-    array([0, 5, 1, 2])
-
-    """
-    try:
-        searchsorted = a.searchsorted
-    except AttributeError:
-        return _wrapit(a, 'searchsorted', v, side, sorter)
-    return searchsorted(v, side, sorter)
-
-
-def resize(a, new_shape):
-    """
-    Return a new array with the specified shape.
-
-    If the new array is larger than the original array, then the new
-    array is filled with repeated copies of `a`.  Note that this behavior
-    is different from a.resize(new_shape) which fills with zeros instead
-    of repeated copies of `a`.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be resized.
-
-    new_shape : int or tuple of int
-        Shape of resized array.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        The new array is formed from the data in the old array, repeated
-        if necessary to fill out the required number of elements.  The
-        data are repeated in the order that they are stored in memory.
-
-    See Also
-    --------
-    ndarray.resize : resize an array in-place.
-
-    Examples
-    --------
-    >>> a=np.array([[0,1],[2,3]])
-    >>> np.resize(a,(1,4))
-    array([[0, 1, 2, 3]])
-    >>> np.resize(a,(2,4))
-    array([[0, 1, 2, 3],
-           [0, 1, 2, 3]])
-
-    """
-    if isinstance(new_shape, (int, nt.integer)):
-        new_shape = (new_shape,)
-    a = ravel(a)
-    Na = len(a)
-    if not Na: return mu.zeros(new_shape, a.dtype.char)
-    total_size = um.multiply.reduce(new_shape)
-    n_copies = int(total_size / Na)
-    extra = total_size % Na
-
-    if total_size == 0:

From noreply at buildbot.pypy.org  Wed Oct 30 17:13:47 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 17:13:47 +0100 (CET)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <20131030161347.D5F381C0330@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67739:9555b1b4162b
Date: 2013-10-30 12:11 -0400
http://bitbucket.org/pypy/pypy/changeset/9555b1b4162b/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -121,3 +121,6 @@
 .. branch: jit-settrace
 Allow the jit to continue running when sys.settrace() is active, necessary to
 make coverage.py fast
+
+.. branch: remove-numpypy
+Remove lib_pypy/numpypy in favor of external numpy fork

From noreply at buildbot.pypy.org  Wed Oct 30 17:13:49 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 17:13:49 +0100 (CET)
Subject: [pypy-commit] pypy remove-numpypy: close merged branch
Message-ID: <20131030161349.0512D1C0330@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: remove-numpypy
Changeset: r67740:9ad0c80d9cff
Date: 2013-10-30 12:12 -0400
http://bitbucket.org/pypy/pypy/changeset/9ad0c80d9cff/

Log:	close merged branch


From noreply at buildbot.pypy.org  Wed Oct 30 17:13:50 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 17:13:50 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20131030161350.5F3421C0330@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67741:403164e5159d
Date: 2013-10-30 12:12 -0400
http://bitbucket.org/pypy/pypy/changeset/403164e5159d/

Log:	merge heads

diff too long, truncating to 2000 out of 7433 lines

diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from . import core
-from .core import *
-from . import lib
-from .lib import *
-
-from __builtin__ import bool, int, long, float, complex, object, unicode, str
-
-from .core import round, abs, max, min
-
-__version__ = '1.7.0'
-
-__all__ = ['__version__']
-__all__ += core.__all__
-__all__ += lib.__all__
-
-#import sys
-#sys.modules.setdefault('numpy', sys.modules['numpypy'])
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from __future__ import division, absolute_import, print_function
-
-from . import multiarray
-from . import umath
-from . import numeric
-from .numeric import *
-from . import fromnumeric
-from .fromnumeric import *
-from . import shape_base
-from .shape_base import *
-
-from .fromnumeric import amax as max, amin as min, \
-    round_ as round
-from .numeric import absolute as abs
-
-__all__ = []
-__all__ += numeric.__all__
-__all__ += fromnumeric.__all__
-__all__ += shape_base.__all__
diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/_methods.py
+++ /dev/null
@@ -1,124 +0,0 @@
-"""
-Array methods which are called by the both the C-code for the method
-and the Python code for the NumPy-namespace function
-
-"""
-from __future__ import division, absolute_import, print_function
-
-import warnings
-
-from . import multiarray as mu
-from . import umath as um
-from .numeric import asanyarray
-from . import numerictypes as nt
-
-def _amax(a, axis=None, out=None, keepdims=False):
-    return um.maximum.reduce(a, axis=axis,
-                            out=out, keepdims=keepdims)
-
-def _amin(a, axis=None, out=None, keepdims=False):
-    return um.minimum.reduce(a, axis=axis,
-                            out=out, keepdims=keepdims)
-
-def _sum(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.add.reduce(a, axis=axis, dtype=dtype,
-                            out=out, keepdims=keepdims)
-
-def _prod(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.multiply.reduce(a, axis=axis, dtype=dtype,
-                            out=out, keepdims=keepdims)
-
-def _any(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.logical_or.reduce(a, axis=axis, dtype=dtype, out=out,
-                                keepdims=keepdims)
-
-def _all(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.logical_and.reduce(a, axis=axis, dtype=dtype, out=out,
-                                 keepdims=keepdims)
-
-def _count_reduce_items(arr, axis):
-    if axis is None:
-        axis = tuple(range(arr.ndim))
-    if not isinstance(axis, tuple):
-        axis = (axis,)
-    items = 1
-    for ax in axis:
-        items *= arr.shape[ax]
-    return items
-
-def _mean(a, axis=None, dtype=None, out=None, keepdims=False):
-    arr = asanyarray(a)
-
-    rcount = _count_reduce_items(arr, axis)
-    # Make this warning show up first
-    if rcount == 0:
-        warnings.warn("Mean of empty slice.", RuntimeWarning)
-
-
-    # Cast bool, unsigned int, and int to float64 by default
-    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
-        dtype = mu.dtype('f8')
-
-    ret = um.add.reduce(arr, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
-    if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(
-                ret, rcount, out=ret, casting='unsafe', subok=False)
-    else:
-        ret = ret.dtype.type(ret / rcount)
-
-    return ret
-
-def _var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
-    arr = asanyarray(a)
-
-    rcount = _count_reduce_items(arr, axis)
-    # Make this warning show up on top.
-    if ddof >= rcount:
-        warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning)
-
-    # Cast bool, unsigned int, and int to float64 by default
-    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
-        dtype = mu.dtype('f8')
-
-    # Compute the mean.
-    # Note that if dtype is not of inexact type then arraymean will
-    # not be either.
-    arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True)
-    if isinstance(arrmean, mu.ndarray):
-        arrmean = um.true_divide(
-                arrmean, rcount, out=arrmean, casting='unsafe', subok=False)
-    else:
-        arrmean = arrmean.dtype.type(arrmean / rcount)
-
-    # Compute sum of squared deviations from mean
-    # Note that x may not be inexact and that we need it to be an array,
-    # not a scalar.
-    x = asanyarray(arr - arrmean)
-    if issubclass(arr.dtype.type, nt.complexfloating):
-        x = um.multiply(x, um.conjugate(x), out=x).real
-    else:
-        x = um.multiply(x, x, out=x)
-    ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
-
-    # Compute degrees of freedom and make sure it is not negative.
-    rcount = max([rcount - ddof, 0])
-
-    # divide by degrees of freedom
-    if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(
-                ret, rcount, out=ret, casting='unsafe', subok=False)
-    else:
-        ret = ret.dtype.type(ret / rcount)
-
-    return ret
-
-def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
-    ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
-               keepdims=keepdims)
-
-    if isinstance(ret, mu.ndarray):
-        ret = um.sqrt(ret, out=ret)
-    else:
-        ret = ret.dtype.type(um.sqrt(ret))
-
-    return ret
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ /dev/null
@@ -1,751 +0,0 @@
-"""Array printing function
-
-$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $
-"""
-__all__ = ["array2string", "set_printoptions", "get_printoptions"]
-__docformat__ = 'restructuredtext'
-
-#
-# Written by Konrad Hinsen <hinsenk at ere.umontreal.ca>
-# last revision: 1996-3-13
-# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details)
-# and by Perry Greenfield 2000-4-1 for numarray
-# and by Travis Oliphant  2005-8-22 for numpy
-
-import sys
-import numerictypes as _nt
-from umath import maximum, minimum, absolute, not_equal, isnan, isinf
-#from multiarray import format_longfloat, datetime_as_string, datetime_data
-from fromnumeric import ravel
-
-
-def product(x, y): return x*y
-
-_summaryEdgeItems = 3     # repr N leading and trailing items of each dimension
-_summaryThreshold = 1000 # total items > triggers array summarization
-
-_float_output_precision = 8
-_float_output_suppress_small = False
-_line_width = 75
-_nan_str = 'nan'
-_inf_str = 'inf'
-_formatter = None  # formatting function for array elements
-
-if sys.version_info[0] >= 3:
-    from functools import reduce
-
-def set_printoptions(precision=None, threshold=None, edgeitems=None,
-                     linewidth=None, suppress=None,
-                     nanstr=None, infstr=None,
-                     formatter=None):
-    """
-    Set printing options.
-
-    These options determine the way floating point numbers, arrays and
-    other NumPy objects are displayed.
-
-    Parameters
-    ----------
-    precision : int, optional
-        Number of digits of precision for floating point output (default 8).
-    threshold : int, optional
-        Total number of array elements which trigger summarization
-        rather than full repr (default 1000).
-    edgeitems : int, optional
-        Number of array items in summary at beginning and end of
-        each dimension (default 3).
-    linewidth : int, optional
-        The number of characters per line for the purpose of inserting
-        line breaks (default 75).
-    suppress : bool, optional
-        Whether or not suppress printing of small floating point values
-        using scientific notation (default False).
-    nanstr : str, optional
-        String representation of floating point not-a-number (default nan).
-    infstr : str, optional
-        String representation of floating point infinity (default inf).
-    formatter : dict of callables, optional
-        If not None, the keys should indicate the type(s) that the respective
-        formatting function applies to.  Callables should return a string.
-        Types that are not specified (by their corresponding keys) are handled
-        by the default formatters.  Individual types for which a formatter
-        can be set are::
-
-            - 'bool'
-            - 'int'
-            - 'timedelta' : a `numpy.timedelta64`
-            - 'datetime' : a `numpy.datetime64`
-            - 'float'
-            - 'longfloat' : 128-bit floats
-            - 'complexfloat'
-            - 'longcomplexfloat' : composed of two 128-bit floats
-            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
-            - 'str' : all other strings
-
-        Other keys that can be used to set a group of types at once are::
-
-            - 'all' : sets all types
-            - 'int_kind' : sets 'int'
-            - 'float_kind' : sets 'float' and 'longfloat'
-            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
-            - 'str_kind' : sets 'str' and 'numpystr'
-
-    See Also
-    --------
-    get_printoptions, set_string_function, array2string
-
-    Notes
-    -----
-    `formatter` is always reset with a call to `set_printoptions`.
-
-    Examples
-    --------
-    Floating point precision can be set:
-
-    >>> np.set_printoptions(precision=4)
-    >>> print np.array([1.123456789])
-    [ 1.1235]
-
-    Long arrays can be summarised:
-
-    >>> np.set_printoptions(threshold=5)
-    >>> print np.arange(10)
-    [0 1 2 ..., 7 8 9]
-
-    Small results can be suppressed:
-
-    >>> eps = np.finfo(float).eps
-    >>> x = np.arange(4.)
-    >>> x**2 - (x + eps)**2
-    array([ -4.9304e-32,  -4.4409e-16,   0.0000e+00,   0.0000e+00])
-    >>> np.set_printoptions(suppress=True)
-    >>> x**2 - (x + eps)**2
-    array([-0., -0.,  0.,  0.])
-
-    A custom formatter can be used to display array elements as desired:
-
-    >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)})
-    >>> x = np.arange(3)
-    >>> x
-    array([int: 0, int: -1, int: -2])
-    >>> np.set_printoptions()  # formatter gets reset
-    >>> x
-    array([0, 1, 2])
-
-    To put back the default options, you can use:
-
-    >>> np.set_printoptions(edgeitems=3,infstr='inf',
-    ... linewidth=75, nanstr='nan', precision=8,
-    ... suppress=False, threshold=1000, formatter=None)
-    """
-
-    global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \
-           _line_width, _float_output_suppress_small, _nan_str, _inf_str, \
-           _formatter
-    if linewidth is not None:
-        _line_width = linewidth
-    if threshold is not None:
-        _summaryThreshold = threshold
-    if edgeitems is not None:
-        _summaryEdgeItems = edgeitems
-    if precision is not None:
-        _float_output_precision = precision
-    if suppress is not None:
-        _float_output_suppress_small = not not suppress
-    if nanstr is not None:
-        _nan_str = nanstr
-    if infstr is not None:
-        _inf_str = infstr
-    _formatter = formatter
-
-def get_printoptions():
-    """
-    Return the current print options.
-
-    Returns
-    -------
-    print_opts : dict
-        Dictionary of current print options with keys
-
-          - precision : int
-          - threshold : int
-          - edgeitems : int
-          - linewidth : int
-          - suppress : bool
-          - nanstr : str
-          - infstr : str
-          - formatter : dict of callables
-
-        For a full description of these options, see `set_printoptions`.
-
-    See Also
-    --------
-    set_printoptions, set_string_function
-
-    """
-    d = dict(precision=_float_output_precision,
-             threshold=_summaryThreshold,
-             edgeitems=_summaryEdgeItems,
-             linewidth=_line_width,
-             suppress=_float_output_suppress_small,
-             nanstr=_nan_str,
-             infstr=_inf_str,
-             formatter=_formatter)
-    return d
-
-def _leading_trailing(a):
-    import numeric as _nc
-    if a.ndim == 1:
-        if len(a) > 2*_summaryEdgeItems:
-            b = _nc.concatenate((a[:_summaryEdgeItems],
-                                     a[-_summaryEdgeItems:]))
-        else:
-            b = a
-    else:
-        if len(a) > 2*_summaryEdgeItems:
-            l = [_leading_trailing(a[i]) for i in range(
-                min(len(a), _summaryEdgeItems))]
-            l.extend([_leading_trailing(a[-i]) for i in range(
-                min(len(a), _summaryEdgeItems),0,-1)])
-        else:
-            l = [_leading_trailing(a[i]) for i in range(0, len(a))]
-        b = _nc.concatenate(tuple(l))
-    return b
-
-def _boolFormatter(x):
-    if x:
-        return ' True'
-    else:
-        return 'False'
-
-
-def repr_format(x):
-    return repr(x)
-
-def _array2string(a, max_line_width, precision, suppress_small, separator=' ',
-                  prefix="", formatter=None):
-
-    if max_line_width is None:
-        max_line_width = _line_width
-
-    if precision is None:
-        precision = _float_output_precision
-
-    if suppress_small is None:
-        suppress_small = _float_output_suppress_small
-
-    if formatter is None:
-        formatter = _formatter
-
-    if a.size > _summaryThreshold:
-        summary_insert = "..., "
-        data = _leading_trailing(a)
-    else:
-        summary_insert = ""
-        data = ravel(a)
-
-    formatdict = {'bool' : _boolFormatter,
-                  'int' : IntegerFormat(data),
-                  'float' : FloatFormat(data, precision, suppress_small),
-                  'longfloat' : FloatFormat(data, precision, suppress_small),
-                  'complexfloat' : ComplexFormat(data, precision,
-                                                 suppress_small),
-                  'longcomplexfloat' : ComplexFormat(data, precision,
-                                                     suppress_small),
-                  'datetime' : DatetimeFormat(data),
-                  'timedelta' : TimedeltaFormat(data),
-                  'numpystr' : repr_format,
-                  'str' : str}
-
-    if formatter is not None:
-        fkeys = [k for k in formatter.keys() if formatter[k] is not None]
-        if 'all' in fkeys:
-            for key in formatdict.keys():
-                formatdict[key] = formatter['all']
-        if 'int_kind' in fkeys:
-            for key in ['int']:
-                formatdict[key] = formatter['int_kind']
-        if 'float_kind' in fkeys:
-            for key in ['float', 'longfloat']:
-                formatdict[key] = formatter['float_kind']
-        if 'complex_kind' in fkeys:
-            for key in ['complexfloat', 'longcomplexfloat']:
-                formatdict[key] = formatter['complex_kind']
-        if 'str_kind' in fkeys:
-            for key in ['numpystr', 'str']:
-                formatdict[key] = formatter['str_kind']
-        for key in formatdict.keys():
-            if key in fkeys:
-                formatdict[key] = formatter[key]
-
-    try:
-        format_function = a._format
-        msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \
-              "will be removed in 2.1. Use the `formatter` kw instead."
-        import warnings
-        warnings.warn(msg, DeprecationWarning)
-    except AttributeError:
-        # find the right formatting function for the array
-        dtypeobj = a.dtype.type
-        if issubclass(dtypeobj, _nt.bool_):
-            format_function = formatdict['bool']
-        elif issubclass(dtypeobj, _nt.integer):
-            #if issubclass(dtypeobj, _nt.timedelta64):
-            #    format_function = formatdict['timedelta']
-            #else:
-            format_function = formatdict['int']
-        elif issubclass(dtypeobj, _nt.floating):
-            if hasattr(_nt, 'longfloat') and issubclass(dtypeobj, _nt.longfloat):
-                format_function = formatdict['longfloat']
-            else:
-                format_function = formatdict['float']
-        elif issubclass(dtypeobj, _nt.complexfloating):
-            if hasattr(_nt, 'clongfloat') and issubclass(dtypeobj, _nt.clongfloat):
-                format_function = formatdict['longcomplexfloat']
-            else:
-                format_function = formatdict['complexfloat']
-        elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)):
-            format_function = formatdict['numpystr']
-        #elif issubclass(dtypeobj, _nt.datetime64):
-        #    format_function = formatdict['datetime']
-        else:
-            format_function = formatdict['str']
-
-    # skip over "["
-    next_line_prefix = " "
-    # skip over array(
-    next_line_prefix += " "*len(prefix)
-
-    lst = _formatArray(a, format_function, len(a.shape), max_line_width,
-                       next_line_prefix, separator,
-                       _summaryEdgeItems, summary_insert)[:-1]
-    return lst
-
-def _convert_arrays(obj):
-    import numeric as _nc
-    newtup = []
-    for k in obj:
-        if isinstance(k, _nc.ndarray):
-            k = k.tolist()
-        elif isinstance(k, tuple):
-            k = _convert_arrays(k)
-        newtup.append(k)
-    return tuple(newtup)
-
-
-def array2string(a, max_line_width=None, precision=None,
-                 suppress_small=None, separator=' ', prefix="",
-                 style=repr, formatter=None):
-    """
-    Return a string representation of an array.
-
-    Parameters
-    ----------
-    a : ndarray
-        Input array.
-    max_line_width : int, optional
-        The maximum number of columns the string should span. Newline
-        characters splits the string appropriately after array elements.
-    precision : int, optional
-        Floating point precision. Default is the current printing
-        precision (usually 8), which can be altered using `set_printoptions`.
-    suppress_small : bool, optional
-        Represent very small numbers as zero. A number is "very small" if it
-        is smaller than the current printing precision.
-    separator : str, optional
-        Inserted between elements.
-    prefix : str, optional
-        An array is typically printed as::
-
-          'prefix(' + array2string(a) + ')'
-
-        The length of the prefix string is used to align the
-        output correctly.
-    style : function, optional
-        A function that accepts an ndarray and returns a string.  Used only
-        when the shape of `a` is equal to ``()``, i.e. for 0-D arrays.
-    formatter : dict of callables, optional
-        If not None, the keys should indicate the type(s) that the respective
-        formatting function applies to.  Callables should return a string.
-        Types that are not specified (by their corresponding keys) are handled
-        by the default formatters.  Individual types for which a formatter
-        can be set are::
-
-            - 'bool'
-            - 'int'
-            - 'timedelta' : a `numpy.timedelta64`
-            - 'datetime' : a `numpy.datetime64`
-            - 'float'
-            - 'longfloat' : 128-bit floats
-            - 'complexfloat'
-            - 'longcomplexfloat' : composed of two 128-bit floats
-            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
-            - 'str' : all other strings
-
-        Other keys that can be used to set a group of types at once are::
-
-            - 'all' : sets all types
-            - 'int_kind' : sets 'int'
-            - 'float_kind' : sets 'float' and 'longfloat'
-            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
-            - 'str_kind' : sets 'str' and 'numpystr'
-
-    Returns
-    -------
-    array_str : str
-        String representation of the array.
-
-    Raises
-    ------
-    TypeError : if a callable in `formatter` does not return a string.
-
-    See Also
-    --------
-    array_str, array_repr, set_printoptions, get_printoptions
-
-    Notes
-    -----
-    If a formatter is specified for a certain type, the `precision` keyword is
-    ignored for that type.
-
-    Examples
-    --------
-    >>> x = np.array([1e-16,1,2,3])
-    >>> print np.array2string(x, precision=2, separator=',',
-    ...                       suppress_small=True)
-    [ 0., 1., 2., 3.]
-
-    >>> x  = np.arange(3.)
-    >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x})
-    '[0.00 1.00 2.00]'
-
-    >>> x  = np.arange(3)
-    >>> np.array2string(x, formatter={'int':lambda x: hex(x)})
-    '[0x0L 0x1L 0x2L]'
-
-    """
-
-    if a.shape == ():
-        x = a.item()
-        try:
-            lst = a._format(x)
-            msg = "The `_format` attribute is deprecated in Numpy " \
-                  "2.0 and will be removed in 2.1. Use the " \
-                  "`formatter` kw instead."
-            import warnings
-            warnings.warn(msg, DeprecationWarning)
-        except AttributeError:
-            if isinstance(x, tuple):
-                x = _convert_arrays(x)
-            lst = style(x)
-    elif reduce(product, a.shape) == 0:
-        # treat as a null array if any of shape elements == 0
-        lst = "[]"
-    else:
-        lst = _array2string(a, max_line_width, precision, suppress_small,
-                            separator, prefix, formatter=formatter)
-    return lst
-
-def _extendLine(s, line, word, max_line_len, next_line_prefix):
-    if len(line.rstrip()) + len(word.rstrip()) >= max_line_len:
-        s += line.rstrip() + "\n"
-        line = next_line_prefix
-    line += word
-    return s, line
-
-
-def _formatArray(a, format_function, rank, max_line_len,
-                 next_line_prefix, separator, edge_items, summary_insert):
-    """formatArray is designed for two modes of operation:
-
-    1. Full output
-
-    2. Summarized output
-
-    """
-    if rank == 0:
-        obj = a.item()
-        if isinstance(obj, tuple):
-            obj = _convert_arrays(obj)
-        return str(obj)
-
-    if summary_insert and 2*edge_items < len(a):
-        leading_items, trailing_items, summary_insert1 = \
-                       edge_items, edge_items, summary_insert
-    else:
-        leading_items, trailing_items, summary_insert1 = 0, len(a), ""
-
-    if rank == 1:
-        s = ""
-        line = next_line_prefix
-        for i in xrange(leading_items):
-            word = format_function(a[i]) + separator
-            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-
-        if summary_insert1:
-            s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix)
-
-        for i in xrange(trailing_items, 1, -1):
-            word = format_function(a[-i]) + separator
-            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-
-        word = format_function(a[-1])
-        s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-        s += line + "]\n"
-        s = '[' + s[len(next_line_prefix):]
-    else:
-        s = '['
-        sep = separator.rstrip()
-        for i in xrange(leading_items):
-            if i > 0:
-                s += next_line_prefix
-            s += _formatArray(a[i], format_function, rank-1, max_line_len,
-                              " " + next_line_prefix, separator, edge_items,
-                              summary_insert)
-            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
-
-        if summary_insert1:
-            s += next_line_prefix + summary_insert1 + "\n"
-
-        for i in xrange(trailing_items, 1, -1):
-            if leading_items or i != trailing_items:
-                s += next_line_prefix
-            s += _formatArray(a[-i], format_function, rank-1, max_line_len,
-                              " " + next_line_prefix, separator, edge_items,
-                              summary_insert)
-            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
-        if leading_items or trailing_items > 1:
-            s += next_line_prefix
-        s += _formatArray(a[-1], format_function, rank-1, max_line_len,
-                          " " + next_line_prefix, separator, edge_items,
-                          summary_insert).rstrip()+']\n'
-    return s
-
-class FloatFormat(object):
-    def __init__(self, data, precision, suppress_small, sign=False):
-        self.precision = precision
-        self.suppress_small = suppress_small
-        self.sign = sign
-        self.exp_format = False
-        self.large_exponent = False
-        self.max_str_len = 0
-        try:
-            self.fillFormat(data)
-        except (TypeError, NotImplementedError):
-            # if reduce(data) fails, this instance will not be called, just
-            # instantiated in formatdict.
-            pass
-
-    def fillFormat(self, data):
-        import numeric as _nc
-        errstate = _nc.seterr(all='ignore')
-        try:
-            special = isnan(data) | isinf(data)
-            valid = not_equal(data, 0) & ~special
-            non_zero = absolute(data.compress(valid))
-            if len(non_zero) == 0:
-                max_val = 0.
-                min_val = 0.
-            else:
-                max_val = maximum.reduce(non_zero)
-                min_val = minimum.reduce(non_zero)
-                if max_val >= 1.e8:
-                    self.exp_format = True
-                if not self.suppress_small and (min_val < 0.0001
-                                           or max_val/min_val > 1000.):
-                    self.exp_format = True
-        finally:
-            _nc.seterr(**errstate)
-
-        if self.exp_format:
-            self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100
-            self.max_str_len = 8 + self.precision
-            if self.large_exponent:
-                self.max_str_len += 1
-            if self.sign:
-                format = '%+'
-            else:
-                format = '%'
-            format = format + '%d.%de' % (self.max_str_len, self.precision)
-        else:
-            format = '%%.%df' % (self.precision,)
-            if len(non_zero):
-                precision = max([_digits(x, self.precision, format)
-                                 for x in non_zero])
-            else:
-                precision = 0
-            precision = min(self.precision, precision)
-            self.max_str_len = len(str(int(max_val))) + precision + 2
-            if _nc.any(special):
-                self.max_str_len = max(self.max_str_len,
-                                       len(_nan_str),
-                                       len(_inf_str)+1)
-            if self.sign:
-                format = '%#+'
-            else:
-                format = '%#'
-            format = format + '%d.%df' % (self.max_str_len, precision)
-
-        self.special_fmt = '%%%ds' % (self.max_str_len,)
-        self.format = format
-
-    def __call__(self, x, strip_zeros=True):
-        import numeric as _nc
-        err = _nc.seterr(invalid='ignore')
-        try:
-            if isnan(x):
-                if self.sign:
-                    return self.special_fmt % ('+' + _nan_str,)
-                else:
-                    return self.special_fmt % (_nan_str,)
-            elif isinf(x):
-                if x > 0:
-                    if self.sign:
-                        return self.special_fmt % ('+' + _inf_str,)
-                    else:
-                        return self.special_fmt % (_inf_str,)
-                else:
-                    return self.special_fmt % ('-' + _inf_str,)
-        finally:
-            _nc.seterr(**err)
-
-        s = self.format % x
-        if self.large_exponent:
-            # 3-digit exponent
-            expsign = s[-3]
-            if expsign == '+' or expsign == '-':
-                s = s[1:-2] + '0' + s[-2:]
-        elif self.exp_format:
-            # 2-digit exponent
-            if s[-3] == '0':
-                s = ' ' + s[:-3] + s[-2:]
-        elif strip_zeros:
-            z = s.rstrip('0')
-            s = z + ' '*(len(s)-len(z))
-        return s
-
-
-def _digits(x, precision, format):
-    s = format % x
-    z = s.rstrip('0')
-    return precision - len(s) + len(z)
-
-
-_MAXINT = sys.maxint
-_MININT = -sys.maxint-1
-class IntegerFormat(object):
-    def __init__(self, data):
-        try:
-            max_str_len = max(len(str(maximum.reduce(data))),
-                              len(str(minimum.reduce(data))))
-            self.format = '%' + str(max_str_len) + 'd'
-        except (TypeError, NotImplementedError):
-            # if reduce(data) fails, this instance will not be called, just
-            # instantiated in formatdict.
-            pass
-        except ValueError:
-            # this occurs when everything is NA
-            pass
-
-    def __call__(self, x):
-        if _MININT < x < _MAXINT:
-            return self.format % x
-        else:
-            return "%s" % x
-
-class LongFloatFormat(object):
-    # XXX Have to add something to determine the width to use a la FloatFormat
-    # Right now, things won't line up properly
-    def __init__(self, precision, sign=False):
-        self.precision = precision
-        self.sign = sign
-
-    def __call__(self, x):
-        if isnan(x):
-            if self.sign:
-                return '+' + _nan_str
-            else:
-                return ' ' + _nan_str
-        elif isinf(x):
-            if x > 0:
-                if self.sign:
-                    return '+' + _inf_str
-                else:
-                    return ' ' + _inf_str
-            else:
-                return '-' + _inf_str
-        elif x >= 0:
-            if self.sign:
-                return '+' + format_longfloat(x, self.precision)
-            else:
-                return ' ' + format_longfloat(x, self.precision)
-        else:
-            return format_longfloat(x, self.precision)
-
-
-class LongComplexFormat(object):
-    def __init__(self, precision):
-        self.real_format = LongFloatFormat(precision)
-        self.imag_format = LongFloatFormat(precision, sign=True)
-
-    def __call__(self, x):
-        r = self.real_format(x.real)
-        i = self.imag_format(x.imag)
-        return r + i + 'j'
-
-
-class ComplexFormat(object):
-    def __init__(self, x, precision, suppress_small):
-        self.real_format = FloatFormat(x.real, precision, suppress_small)
-        self.imag_format = FloatFormat(x.imag, precision, suppress_small,
-                                       sign=True)
-
-    def __call__(self, x):
-        r = self.real_format(x.real, strip_zeros=False)
-        i = self.imag_format(x.imag, strip_zeros=False)
-        if not self.imag_format.exp_format:
-            z = i.rstrip('0')
-            i = z + 'j' + ' '*(len(i)-len(z))
-        else:
-            i = i + 'j'
-        return r + i
-
-class DatetimeFormat(object):
-    def __init__(self, x, unit=None,
-                timezone=None, casting='same_kind'):
-        # Get the unit from the dtype
-        if unit is None:
-            if x.dtype.kind == 'M':
-                unit = datetime_data(x.dtype)[0]
-            else:
-                unit = 's'
-
-        # If timezone is default, make it 'local' or 'UTC' based on the unit
-        if timezone is None:
-            # Date units -> UTC, time units -> local
-            if unit in ('Y', 'M', 'W', 'D'):
-                self.timezone = 'UTC'
-            else:
-                self.timezone = 'local'
-        else:
-            self.timezone = timezone
-        self.unit = unit
-        self.casting = casting
-
-    def __call__(self, x):
-        return "'%s'" % datetime_as_string(x,
-                                    unit=self.unit,
-                                    timezone=self.timezone,
-                                    casting=self.casting)
-
-class TimedeltaFormat(object):
-    def __init__(self, data):
-        if data.dtype.kind == 'm':
-            v = data.view('i8')
-            max_str_len = max(len(str(maximum.reduce(v))),
-                              len(str(minimum.reduce(v))))
-            self.format = '%' + str(max_str_len) + 'd'
-
-    def __call__(self, x):
-        return self.format % x.astype('i8')
-
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ /dev/null
@@ -1,2924 +0,0 @@
-######################################################################
-# This is a copy of numpy/core/fromnumeric.py modified for numpypy
-######################################################################
-"""Module containing non-deprecated functions borrowed from Numeric.
-
-"""
-from __future__ import division, absolute_import, print_function
-
-import types
-
-from . import multiarray as mu
-from . import umath as um
-from . import numerictypes as nt
-from .numeric import asarray, array, asanyarray, concatenate
-from . import _methods
-
-
-# functions that are methods
-__all__ = [
-        'alen', 'all', 'alltrue', 'amax', 'amin', 'any', 'argmax',
-        'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip',
-        'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean',
-        'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put',
-        'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_',
-        'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze',
-        'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var',
-        ]
-
-
-try:
-    _gentype = types.GeneratorType
-except AttributeError:
-    _gentype = type(None)
-
-# save away Python sum
-_sum_ = sum
-
-# functions that are now methods
-def _wrapit(obj, method, *args, **kwds):
-    try:
-        wrap = obj.__array_wrap__
-    except AttributeError:
-        wrap = None
-    result = getattr(asarray(obj), method)(*args, **kwds)
-    if wrap:
-        if not isinstance(result, mu.ndarray):
-            result = asarray(result)
-        result = wrap(result)
-    return result
-
-
-def take(a, indices, axis=None, out=None, mode='raise'):
-    """
-    Take elements from an array along an axis.
-
-    This function does the same thing as "fancy" indexing (indexing arrays
-    using arrays); however, it can be easier to use if you need elements
-    along a given axis.
-
-    Parameters
-    ----------
-    a : array_like
-        The source array.
-    indices : array_like
-        The indices of the values to extract.
-
-        .. versionadded:: 1.8.0
-
-        Also allow scalars for indices.
-    axis : int, optional
-        The axis over which to select values. By default, the flattened
-        input array is used.
-    out : ndarray, optional
-        If provided, the result will be placed in this array. It should
-        be of the appropriate shape and dtype.
-    mode : {'raise', 'wrap', 'clip'}, optional
-        Specifies how out-of-bounds indices will behave.
-
-        * 'raise' -- raise an error (default)
-        * 'wrap' -- wrap around
-        * 'clip' -- clip to the range
-
-        'clip' mode means that all indices that are too large are replaced
-        by the index that addresses the last element along that axis. Note
-        that this disables indexing with negative numbers.
-
-    Returns
-    -------
-    subarray : ndarray
-        The returned array has the same type as `a`.
-
-    See Also
-    --------
-    ndarray.take : equivalent method
-
-    Examples
-    --------
-    >>> a = [4, 3, 5, 7, 6, 8]
-    >>> indices = [0, 1, 4]
-    >>> np.take(a, indices)
-    array([4, 3, 6])
-
-    In this example if `a` is an ndarray, "fancy" indexing can be used.
-
-    >>> a = np.array(a)
-    >>> a[indices]
-    array([4, 3, 6])
-
-    If `indices` is not one dimensional, the output also has these dimensions.
-
-    >>> np.take(a, [[0, 1], [2, 3]])
-    array([[4, 3],
-           [5, 7]])
-    """
-    try:
-        take = a.take
-    except AttributeError:
-        return _wrapit(a, 'take', indices, axis, out, mode)
-    return take(indices, axis, out, mode)
-
-
-# not deprecated --- copy if necessary, view otherwise
-def reshape(a, newshape, order='C'):
-    """
-    Gives a new shape to an array without changing its data.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be reshaped.
-    newshape : int or tuple of ints
-        The new shape should be compatible with the original shape. If
-        an integer, then the result will be a 1-D array of that length.
-        One shape dimension can be -1. In this case, the value is inferred
-        from the length of the array and remaining dimensions.
-    order : {'C', 'F', 'A'}, optional
-        Read the elements of `a` using this index order, and place the elements
-        into the reshaped array using this index order.  'C' means to
-        read / write the elements using C-like index order, with the last axis index
-        changing fastest, back to the first axis index changing slowest.  'F'
-        means to read / write the elements using Fortran-like index order, with
-        the first index changing fastest, and the last index changing slowest.
-        Note that the 'C' and 'F' options take no account of the memory layout
-        of the underlying array, and only refer to the order of indexing.  'A'
-        means to read / write the elements in Fortran-like index order if `a` is
-        Fortran *contiguous* in memory, C-like order otherwise.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        This will be a new view object if possible; otherwise, it will
-        be a copy.  Note there is no guarantee of the *memory layout* (C- or
-        Fortran- contiguous) of the returned array.
-
-    See Also
-    --------
-    ndarray.reshape : Equivalent method.
-
-    Notes
-    -----
-    It is not always possible to change the shape of an array without
-    copying the data. If you want an error to be raise if the data is copied,
-    you should assign the new shape to the shape attribute of the array::
-
-     >>> a = np.zeros((10, 2))
-     # A transpose make the array non-contiguous
-     >>> b = a.T
-     # Taking a view makes it possible to modify the shape without modifying the
-     # initial object.
-     >>> c = b.view()
-     >>> c.shape = (20)
-     AttributeError: incompatible shape for a non-contiguous array
-
-    The `order` keyword gives the index ordering both for *fetching* the values
-    from `a`, and then *placing* the values into the output array.  For example,
-    let's say you have an array:
-
-    >>> a = np.arange(6).reshape((3, 2))
-    >>> a
-    array([[0, 1],
-           [2, 3],
-           [4, 5]])
-
-    You can think of reshaping as first raveling the array (using the given
-    index order), then inserting the elements from the raveled array into the
-    new array using the same kind of index ordering as was used for the
-    raveling.
-
-    >>> np.reshape(a, (2, 3)) # C-like index ordering
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.reshape(a, (2, 3), order='F') # Fortran-like index ordering
-    array([[0, 4, 3],
-           [2, 1, 5]])
-    >>> np.reshape(np.ravel(a, order='F'), (2, 3), order='F')
-    array([[0, 4, 3],
-           [2, 1, 5]])
-
-    Examples
-    --------
-    >>> a = np.array([[1,2,3], [4,5,6]])
-    >>> np.reshape(a, 6)
-    array([1, 2, 3, 4, 5, 6])
-    >>> np.reshape(a, 6, order='F')
-    array([1, 4, 2, 5, 3, 6])
-
-    >>> np.reshape(a, (3,-1))       # the unspecified value is inferred to be 2
-    array([[1, 2],
-           [3, 4],
-           [5, 6]])
-    """
-    assert order == 'C'
-    try:
-        reshape = a.reshape
-    except AttributeError:
-        return _wrapit(a, 'reshape', newshape)
-    return reshape(newshape)
-
-
-def choose(a, choices, out=None, mode='raise'):
-    """
-    Construct an array from an index array and a set of arrays to choose from.
-
-    First of all, if confused or uncertain, definitely look at the Examples -
-    in its full generality, this function is less simple than it might
-    seem from the following code description (below ndi =
-    `numpy.lib.index_tricks`):
-
-    ``np.choose(a,c) == np.array([c[a[I]][I] for I in ndi.ndindex(a.shape)])``.
-
-    But this omits some subtleties.  Here is a fully general summary:
-
-    Given an "index" array (`a`) of integers and a sequence of `n` arrays
-    (`choices`), `a` and each choice array are first broadcast, as necessary,
-    to arrays of a common shape; calling these *Ba* and *Bchoices[i], i =
-    0,...,n-1* we have that, necessarily, ``Ba.shape == Bchoices[i].shape``
-    for each `i`.  Then, a new array with shape ``Ba.shape`` is created as
-    follows:
-
-    * if ``mode=raise`` (the default), then, first of all, each element of
-      `a` (and thus `Ba`) must be in the range `[0, n-1]`; now, suppose that
-      `i` (in that range) is the value at the `(j0, j1, ..., jm)` position
-      in `Ba` - then the value at the same position in the new array is the
-      value in `Bchoices[i]` at that same position;
-
-    * if ``mode=wrap``, values in `a` (and thus `Ba`) may be any (signed)
-      integer; modular arithmetic is used to map integers outside the range
-      `[0, n-1]` back into that range; and then the new array is constructed
-      as above;
-
-    * if ``mode=clip``, values in `a` (and thus `Ba`) may be any (signed)
-      integer; negative integers are mapped to 0; values greater than `n-1`
-      are mapped to `n-1`; and then the new array is constructed as above.
-
-    Parameters
-    ----------
-    a : int array
-        This array must contain integers in `[0, n-1]`, where `n` is the number
-        of choices, unless ``mode=wrap`` or ``mode=clip``, in which cases any
-        integers are permissible.
-    choices : sequence of arrays
-        Choice arrays. `a` and all of the choices must be broadcastable to the
-        same shape.  If `choices` is itself an array (not recommended), then
-        its outermost dimension (i.e., the one corresponding to
-        ``choices.shape[0]``) is taken as defining the "sequence".
-    out : array, optional
-        If provided, the result will be inserted into this array. It should
-        be of the appropriate shape and dtype.
-    mode : {'raise' (default), 'wrap', 'clip'}, optional
-        Specifies how indices outside `[0, n-1]` will be treated:
-
-          * 'raise' : an exception is raised
-          * 'wrap' : value becomes value mod `n`
-          * 'clip' : values < 0 are mapped to 0, values > n-1 are mapped to n-1
-
-    Returns
-    -------
-    merged_array : array
-        The merged result.
-
-    Raises
-    ------
-    ValueError: shape mismatch
-        If `a` and each choice array are not all broadcastable to the same
-        shape.
-
-    See Also
-    --------
-    ndarray.choose : equivalent method
-
-    Notes
-    -----
-    To reduce the chance of misinterpretation, even though the following
-    "abuse" is nominally supported, `choices` should neither be, nor be
-    thought of as, a single array, i.e., the outermost sequence-like container
-    should be either a list or a tuple.
-
-    Examples
-    --------
-
-    >>> choices = [[0, 1, 2, 3], [10, 11, 12, 13],
-    ...   [20, 21, 22, 23], [30, 31, 32, 33]]
-    >>> np.choose([2, 3, 1, 0], choices
-    ... # the first element of the result will be the first element of the
-    ... # third (2+1) "array" in choices, namely, 20; the second element
-    ... # will be the second element of the fourth (3+1) choice array, i.e.,
-    ... # 31, etc.
-    ... )
-    array([20, 31, 12,  3])
-    >>> np.choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1)
-    array([20, 31, 12,  3])
-    >>> # because there are 4 choice arrays
-    >>> np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4)
-    array([20,  1, 12,  3])
-    >>> # i.e., 0
-
-    A couple examples illustrating how choose broadcasts:
-
-    >>> a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]]
-    >>> choices = [-10, 10]
-    >>> np.choose(a, choices)
-    array([[ 10, -10,  10],
-           [-10,  10, -10],
-           [ 10, -10,  10]])
-
-    >>> # With thanks to Anne Archibald
-    >>> a = np.array([0, 1]).reshape((2,1,1))
-    >>> c1 = np.array([1, 2, 3]).reshape((1,3,1))
-    >>> c2 = np.array([-1, -2, -3, -4, -5]).reshape((1,1,5))
-    >>> np.choose(a, (c1, c2)) # result is 2x3x5, res[0,:,:]=c1, res[1,:,:]=c2
-    array([[[ 1,  1,  1,  1,  1],
-            [ 2,  2,  2,  2,  2],
-            [ 3,  3,  3,  3,  3]],
-           [[-1, -2, -3, -4, -5],
-            [-1, -2, -3, -4, -5],
-            [-1, -2, -3, -4, -5]]])
-
-    """
-    try:
-        choose = a.choose
-    except AttributeError:
-        return _wrapit(a, 'choose', choices, out=out, mode=mode)
-    return choose(choices, out=out, mode=mode)
-
-
-def repeat(a, repeats, axis=None):
-    """
-    Repeat elements of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    repeats : {int, array of ints}
-        The number of repetitions for each element.  `repeats` is broadcasted
-        to fit the shape of the given axis.
-    axis : int, optional
-        The axis along which to repeat values.  By default, use the
-        flattened input array, and return a flat output array.
-
-    Returns
-    -------
-    repeated_array : ndarray
-        Output array which has the same shape as `a`, except along
-        the given axis.
-
-    See Also
-    --------
-    tile : Tile an array.
-
-    Examples
-    --------
-    >>> x = np.array([[1,2],[3,4]])
-    >>> np.repeat(x, 2)
-    array([1, 1, 2, 2, 3, 3, 4, 4])
-    >>> np.repeat(x, 3, axis=1)
-    array([[1, 1, 1, 2, 2, 2],
-           [3, 3, 3, 4, 4, 4]])
-    >>> np.repeat(x, [1, 2], axis=0)
-    array([[1, 2],
-           [3, 4],
-           [3, 4]])
-
-    """
-    try:
-        repeat = a.repeat
-    except AttributeError:
-        return _wrapit(a, 'repeat', repeats, axis)
-    return repeat(repeats, axis)
-
-
-def put(a, ind, v, mode='raise'):
-    """
-    Replaces specified elements of an array with given values.
-
-    The indexing works on the flattened target array. `put` is roughly
-    equivalent to:
-
-    ::
-
-        a.flat[ind] = v
-
-    Parameters
-    ----------
-    a : ndarray
-        Target array.
-    ind : array_like
-        Target indices, interpreted as integers.
-    v : array_like
-        Values to place in `a` at target indices. If `v` is shorter than
-        `ind` it will be repeated as necessary.
-    mode : {'raise', 'wrap', 'clip'}, optional
-        Specifies how out-of-bounds indices will behave.
-
-        * 'raise' -- raise an error (default)
-        * 'wrap' -- wrap around
-        * 'clip' -- clip to the range
-
-        'clip' mode means that all indices that are too large are replaced
-        by the index that addresses the last element along that axis. Note
-        that this disables indexing with negative numbers.
-
-    See Also
-    --------
-    putmask, place
-
-    Examples
-    --------
-    >>> a = np.arange(5)
-    >>> np.put(a, [0, 2], [-44, -55])
-    >>> a
-    array([-44,   1, -55,   3,   4])
-
-    >>> a = np.arange(5)
-    >>> np.put(a, 22, -5, mode='clip')
-    >>> a
-    array([ 0,  1,  2,  3, -5])
-
-    """
-    return a.put(ind, v, mode)
-
-
-def swapaxes(a, axis1, axis2):
-    """
-    Interchange two axes of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axis1 : int
-        First axis.
-    axis2 : int
-        Second axis.
-
-    Returns
-    -------
-    a_swapped : ndarray
-        If `a` is an ndarray, then a view of `a` is returned; otherwise
-        a new array is created.
-
-    Examples
-    --------
-    >>> x = np.array([[1,2,3]])
-    >>> np.swapaxes(x,0,1)
-    array([[1],
-           [2],
-           [3]])
-
-    >>> x = np.array([[[0,1],[2,3]],[[4,5],[6,7]]])
-    >>> x
-    array([[[0, 1],
-            [2, 3]],
-           [[4, 5],
-            [6, 7]]])
-
-    >>> np.swapaxes(x,0,2)
-    array([[[0, 4],
-            [2, 6]],
-           [[1, 5],
-            [3, 7]]])
-
-    """
-    try:
-        swapaxes = a.swapaxes
-    except AttributeError:
-        return _wrapit(a, 'swapaxes', axis1, axis2)
-    return swapaxes(axis1, axis2)
-
-
-def transpose(a, axes=None):
-    """
-    Permute the dimensions of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axes : list of ints, optional
-        By default, reverse the dimensions, otherwise permute the axes
-        according to the values given.
-
-    Returns
-    -------
-    p : ndarray
-        `a` with its axes permuted.  A view is returned whenever
-        possible.
-
-    See Also
-    --------
-    rollaxis
-
-    Examples
-    --------
-    >>> x = np.arange(4).reshape((2,2))
-    >>> x
-    array([[0, 1],
-           [2, 3]])
-
-    >>> np.transpose(x)
-    array([[0, 2],
-           [1, 3]])
-
-    >>> x = np.ones((1, 2, 3))
-    >>> np.transpose(x, (1, 0, 2)).shape
-    (2, 1, 3)
-
-    """
-    if axes is not None:
-        raise NotImplementedError('No "axes" arg yet.')
-    try:
-        transpose = a.transpose
-    except AttributeError:
-        return _wrapit(a, 'transpose')
-    return transpose()
-
-
-def partition(a, kth, axis=-1, kind='introselect', order=None):
-    """
-    Return a partitioned copy of an array.
-
-    Creates a copy of the array with its elements rearranged in such a way that
-    the value of the element in kth position is in the position it would be in
-    a sorted array. All elements smaller than the kth element are moved before
-    this element and all equal or greater are moved behind it. The ordering of
-    the elements in the two partitions is undefined.
-
-    .. versionadded:: 1.8.0
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be sorted.
-    kth : int or sequence of ints
-        Element index to partition by. The kth value of the element will be in
-        its final sorted position and all smaller elements will be moved before
-        it and all equal or greater elements behind it.
-        The order all elements in the partitions is undefined.
-        If provided with a sequence of kth it will partition all elements
-        indexed by kth  of them into their sorted position at once.
-    axis : int or None, optional
-        Axis along which to sort. If None, the array is flattened before
-        sorting. The default is -1, which sorts along the last axis.
-    kind : {'introselect'}, optional
-        Selection algorithm. Default is 'introselect'.
-    order : list, optional
-        When `a` is a structured array, this argument specifies which fields
-        to compare first, second, and so on.  This list does not need to
-        include all of the fields.
-
-    Returns
-    -------
-    partitioned_array : ndarray
-        Array of the same type and shape as `a`.
-
-    See Also
-    --------
-    ndarray.partition : Method to sort an array in-place.
-    argpartition : Indirect partition.
-    sort : Full sorting
-
-    Notes
-    -----
-    The various selection algorithms are characterized by their average speed,
-    worst case performance, work space size, and whether they are stable. A
-    stable sort keeps items with the same key in the same relative order. The
-    three available algorithms have the following properties:
-
-    ================= ======= ============= ============ =======
-       kind            speed   worst case    work space  stable
-    ================= ======= ============= ============ =======
-    'introselect'        1        O(n)           0         no
-    ================= ======= ============= ============ =======
-
-    All the partition algorithms make temporary copies of the data when
-    partitioning along any but the last axis.  Consequently, partitioning
-    along the last axis is faster and uses less space than partitioning
-    along any other axis.
-
-    The sort order for complex numbers is lexicographic. If both the real
-    and imaginary parts are non-nan then the order is determined by the
-    real parts except when they are equal, in which case the order is
-    determined by the imaginary parts.
-
-    Examples
-    --------
-    >>> a = np.array([3, 4, 2, 1])
-    >>> np.partition(a, 3)
-    array([2, 1, 3, 4])
-
-    >>> np.partition(a, (1, 3))
-    array([1, 2, 3, 4])
-
-    """
-    if axis is None:
-        a = asanyarray(a).flatten()
-        axis = 0
-    else:
-        a = asanyarray(a).copy()
-    a.partition(kth, axis=axis, kind=kind, order=order)
-    return a
-
-
-def argpartition(a, kth, axis=-1, kind='introselect', order=None):
-    """
-    Perform an indirect partition along the given axis using the algorithm
-    specified by the `kind` keyword. It returns an array of indices of the
-    same shape as `a` that index data along the given axis in partitioned
-    order.
-
-    .. versionadded:: 1.8.0
-
-    Parameters
-    ----------
-    a : array_like
-        Array to sort.
-    kth : int or sequence of ints
-        Element index to partition by. The kth element will be in its final
-        sorted position and all smaller elements will be moved before it and
-        all larger elements behind it.
-        The order all elements in the partitions is undefined.
-        If provided with a sequence of kth it will partition all of them into
-        their sorted position at once.
-    axis : int or None, optional
-        Axis along which to sort.  The default is -1 (the last axis). If None,
-        the flattened array is used.
-    kind : {'introselect'}, optional
-        Selection algorithm. Default is 'introselect'
-    order : list, optional
-        When `a` is an array with fields defined, this argument specifies
-        which fields to compare first, second, etc.  Not all fields need be
-        specified.
-
-    Returns
-    -------
-    index_array : ndarray, int
-        Array of indices that partition `a` along the specified axis.
-        In other words, ``a[index_array]`` yields a sorted `a`.
-
-    See Also
-    --------
-    partition : Describes partition algorithms used.
-    ndarray.partition : Inplace partition.
-    argsort : Full indirect sort
-
-    Notes
-    -----
-    See `partition` for notes on the different selection algorithms.
-
-    Examples
-    --------
-    One dimensional array:
-
-    >>> x = np.array([3, 4, 2, 1])
-    >>> x[np.argpartition(x, 3)]
-    array([2, 1, 3, 4])
-    >>> x[np.argpartition(x, (1, 3))]
-    array([1, 2, 3, 4])
-
-    """
-    return a.argpartition(kth, axis, kind=kind, order=order)
-
-
-def sort(a, axis=-1, kind='quicksort', order=None):
-    """
-    Return a sorted copy of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be sorted.
-    axis : int or None, optional
-        Axis along which to sort. If None, the array is flattened before
-        sorting. The default is -1, which sorts along the last axis.
-    kind : {'quicksort', 'mergesort', 'heapsort'}, optional
-        Sorting algorithm. Default is 'quicksort'.
-    order : list, optional
-        When `a` is a structured array, this argument specifies which fields
-        to compare first, second, and so on.  This list does not need to
-        include all of the fields.
-
-    Returns
-    -------
-    sorted_array : ndarray
-        Array of the same type and shape as `a`.
-
-    See Also
-    --------
-    ndarray.sort : Method to sort an array in-place.
-    argsort : Indirect sort.
-    lexsort : Indirect stable sort on multiple keys.
-    searchsorted : Find elements in a sorted array.
-    partition : Partial sort.
-
-    Notes
-    -----
-    The various sorting algorithms are characterized by their average speed,
-    worst case performance, work space size, and whether they are stable. A
-    stable sort keeps items with the same key in the same relative
-    order. The three available algorithms have the following
-    properties:
-
-    =========== ======= ============= ============ =======
-       kind      speed   worst case    work space  stable
-    =========== ======= ============= ============ =======
-    'quicksort'    1     O(n^2)            0          no
-    'mergesort'    2     O(n*log(n))      ~n/2        yes
-    'heapsort'     3     O(n*log(n))       0          no
-    =========== ======= ============= ============ =======
-
-    All the sort algorithms make temporary copies of the data when
-    sorting along any but the last axis.  Consequently, sorting along
-    the last axis is faster and uses less space than sorting along
-    any other axis.
-
-    The sort order for complex numbers is lexicographic. If both the real
-    and imaginary parts are non-nan then the order is determined by the
-    real parts except when they are equal, in which case the order is
-    determined by the imaginary parts.
-
-    Previous to numpy 1.4.0 sorting real and complex arrays containing nan
-    values led to undefined behaviour. In numpy versions >= 1.4.0 nan
-    values are sorted to the end. The extended sort order is:
-
-      * Real: [R, nan]
-      * Complex: [R + Rj, R + nanj, nan + Rj, nan + nanj]
-
-    where R is a non-nan real value. Complex values with the same nan
-    placements are sorted according to the non-nan part if it exists.
-    Non-nan values are sorted as before.
-
-    Examples
-    --------
-    >>> a = np.array([[1,4],[3,1]])
-    >>> np.sort(a)                # sort along the last axis
-    array([[1, 4],
-           [1, 3]])
-    >>> np.sort(a, axis=None)     # sort the flattened array
-    array([1, 1, 3, 4])
-    >>> np.sort(a, axis=0)        # sort along the first axis
-    array([[1, 1],
-           [3, 4]])
-
-    Use the `order` keyword to specify a field to use when sorting a
-    structured array:
-
-    >>> dtype = [('name', 'S10'), ('height', float), ('age', int)]
-    >>> values = [('Arthur', 1.8, 41), ('Lancelot', 1.9, 38),
-    ...           ('Galahad', 1.7, 38)]
-    >>> a = np.array(values, dtype=dtype)       # create a structured array
-    >>> np.sort(a, order='height')                        # doctest: +SKIP
-    array([('Galahad', 1.7, 38), ('Arthur', 1.8, 41),
-           ('Lancelot', 1.8999999999999999, 38)],
-          dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
-
-    Sort by age, then height if ages are equal:
-
-    >>> np.sort(a, order=['age', 'height'])               # doctest: +SKIP
-    array([('Galahad', 1.7, 38), ('Lancelot', 1.8999999999999999, 38),
-           ('Arthur', 1.8, 41)],
-          dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
-
-    """
-    if axis is None:
-        a = asanyarray(a).flatten()
-        axis = 0
-    else:
-        a = asanyarray(a).copy()
-    a.sort(axis, kind, order)
-    return a
-
-
-def argsort(a, axis=-1, kind='quicksort', order=None):
-    """
-    Returns the indices that would sort an array.
-
-    Perform an indirect sort along the given axis using the algorithm specified
-    by the `kind` keyword. It returns an array of indices of the same shape as
-    `a` that index data along the given axis in sorted order.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to sort.
-    axis : int or None, optional
-        Axis along which to sort.  The default is -1 (the last axis). If None,
-        the flattened array is used.
-    kind : {'quicksort', 'mergesort', 'heapsort'}, optional
-        Sorting algorithm.
-    order : list, optional
-        When `a` is an array with fields defined, this argument specifies
-        which fields to compare first, second, etc.  Not all fields need be
-        specified.
-
-    Returns
-    -------
-    index_array : ndarray, int
-        Array of indices that sort `a` along the specified axis.
-        In other words, ``a[index_array]`` yields a sorted `a`.
-
-    See Also
-    --------
-    sort : Describes sorting algorithms used.
-    lexsort : Indirect stable sort with multiple keys.
-    ndarray.sort : Inplace sort.
-    argpartition : Indirect partial sort.
-
-    Notes
-    -----
-    See `sort` for notes on the different sorting algorithms.
-
-    As of NumPy 1.4.0 `argsort` works with real/complex arrays containing
-    nan values. The enhanced sort order is documented in `sort`.
-
-    Examples
-    --------
-    One dimensional array:
-
-    >>> x = np.array([3, 1, 2])
-    >>> np.argsort(x)
-    array([1, 2, 0])
-
-    Two-dimensional array:
-
-    >>> x = np.array([[0, 3], [2, 2]])
-    >>> x
-    array([[0, 3],
-           [2, 2]])
-
-    >>> np.argsort(x, axis=0)
-    array([[0, 1],
-           [1, 0]])
-
-    >>> np.argsort(x, axis=1)
-    array([[0, 1],
-           [0, 1]])
-
-    Sorting with keys:
-
-    >>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '<i4'), ('y', '<i4')])
-    >>> x
-    array([(1, 0), (0, 1)],
-          dtype=[('x', '<i4'), ('y', '<i4')])
-
-    >>> np.argsort(x, order=('x','y'))
-    array([1, 0])
-
-    >>> np.argsort(x, order=('y','x'))
-    array([0, 1])
-
-    """
-    try:
-        argsort = a.argsort
-    except AttributeError:
-        return _wrapit(a, 'argsort', axis, kind, order)
-    return argsort(axis, kind, order)
-
-
-def argmax(a, axis=None):
-    """
-    Indices of the maximum values along an axis.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axis : int, optional
-        By default, the index is into the flattened array, otherwise
-        along the specified axis.
-
-    Returns
-    -------
-    index_array : ndarray of ints
-        Array of indices into the array. It has the same shape as `a.shape`
-        with the dimension along `axis` removed.
-
-    See Also
-    --------
-    ndarray.argmax, argmin
-    amax : The maximum value along a given axis.
-    unravel_index : Convert a flat index into an index tuple.
-
-    Notes
-    -----
-    In case of multiple occurrences of the maximum values, the indices
-    corresponding to the first occurrence are returned.
-
-    Examples
-    --------
-    >>> a = np.arange(6).reshape(2,3)
-    >>> a
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.argmax(a)
-    5
-    >>> np.argmax(a, axis=0)
-    array([1, 1, 1])
-    >>> np.argmax(a, axis=1)
-    array([2, 2])
-
-    >>> b = np.arange(6)
-    >>> b[1] = 5
-    >>> b
-    array([0, 5, 2, 3, 4, 5])
-    >>> np.argmax(b) # Only the first occurrence is returned.
-    1
-
-    """
-    assert axis is None
-    try:
-        argmax = a.argmax
-    except AttributeError:
-        return _wrapit(a, 'argmax')
-    return argmax()
-
-
-def argmin(a, axis=None):
-    """
-    Return the indices of the minimum values along an axis.
-
-    See Also
-    --------
-    argmax : Similar function.  Please refer to `numpy.argmax` for detailed
-        documentation.
-
-    """
-    assert axis is None
-    try:
-        argmin = a.argmin
-    except AttributeError:
-        return _wrapit(a, 'argmin')
-    return argmin()
-
-
-def searchsorted(a, v, side='left', sorter=None):
-    """
-    Find indices where elements should be inserted to maintain order.
-
-    Find the indices into a sorted array `a` such that, if the
-    corresponding elements in `v` were inserted before the indices, the
-    order of `a` would be preserved.
-
-    Parameters
-    ----------
-    a : 1-D array_like
-        Input array. If `sorter` is None, then it must be sorted in
-        ascending order, otherwise `sorter` must be an array of indices
-        that sort it.
-    v : array_like
-        Values to insert into `a`.
-    side : {'left', 'right'}, optional
-        If 'left', the index of the first suitable location found is given.
-        If 'right', return the last such index.  If there is no suitable
-        index, return either 0 or N (where N is the length of `a`).
-    sorter : 1-D array_like, optional
-        .. versionadded:: 1.7.0
-        Optional array of integer indices that sort array a into ascending
-        order. They are typically the result of argsort.
-
-    Returns
-    -------
-    indices : array of ints
-        Array of insertion points with the same shape as `v`.
-
-    See Also
-    --------
-    sort : Return a sorted copy of an array.
-    histogram : Produce histogram from 1-D data.
-
-    Notes
-    -----
-    Binary search is used to find the required insertion points.
-
-    As of Numpy 1.4.0 `searchsorted` works with real/complex arrays containing
-    `nan` values. The enhanced sort order is documented in `sort`.
-
-    Examples
-    --------
-    >>> np.searchsorted([1,2,3,4,5], 3)
-    2
-    >>> np.searchsorted([1,2,3,4,5], 3, side='right')
-    3
-    >>> np.searchsorted([1,2,3,4,5], [-10, 10, 2, 3])
-    array([0, 5, 1, 2])
-
-    """
-    try:
-        searchsorted = a.searchsorted
-    except AttributeError:
-        return _wrapit(a, 'searchsorted', v, side, sorter)
-    return searchsorted(v, side, sorter)
-
-
-def resize(a, new_shape):
-    """
-    Return a new array with the specified shape.
-
-    If the new array is larger than the original array, then the new
-    array is filled with repeated copies of `a`.  Note that this behavior
-    is different from a.resize(new_shape) which fills with zeros instead
-    of repeated copies of `a`.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be resized.
-
-    new_shape : int or tuple of int
-        Shape of resized array.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        The new array is formed from the data in the old array, repeated
-        if necessary to fill out the required number of elements.  The
-        data are repeated in the order that they are stored in memory.
-
-    See Also
-    --------
-    ndarray.resize : resize an array in-place.
-
-    Examples
-    --------
-    >>> a=np.array([[0,1],[2,3]])
-    >>> np.resize(a,(1,4))
-    array([[0, 1, 2, 3]])
-    >>> np.resize(a,(2,4))
-    array([[0, 1, 2, 3],
-           [0, 1, 2, 3]])
-
-    """
-    if isinstance(new_shape, (int, nt.integer)):
-        new_shape = (new_shape,)
-    a = ravel(a)
-    Na = len(a)
-    if not Na: return mu.zeros(new_shape, a.dtype.char)
-    total_size = um.multiply.reduce(new_shape)
-    n_copies = int(total_size / Na)
-    extra = total_size % Na
-
-    if total_size == 0:

From noreply at buildbot.pypy.org  Wed Oct 30 18:02:07 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 30 Oct 2013 18:02:07 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: in-progress
Message-ID: <20131030170207.58E811C13E2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67742:cebfa3182b4d
Date: 2013-10-30 18:01 +0100
http://bitbucket.org/pypy/pypy/changeset/cebfa3182b4d/

Log:	in-progress

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -1,14 +1,23 @@
-from rpython.rlib.rarithmetic import r_singlefloat
+from rpython.rlib.rarithmetic import r_singlefloat, intmask
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 
+r_uint32 = rffi.r_uint
+assert r_uint32.BITS == 32
+UINT32MAX = 2 ** 32 - 1
+
+
 class JitCounter:
     DEFAULT_SIZE = 4096
 
     def __init__(self, size=DEFAULT_SIZE):
-        assert size >= 1 and (size & (size - 1)) == 0     # a power of two
-        self.mask = size - 1
+        "NOT_RPYTHON"
+        self.size = size
+        self.shift = 1
+        while (UINT32MAX >> self.shift) != size - 1:
+            self.shift += 1
+            assert self.shift < 999, "size is not a power of two <= 2**31"
         self.timetable = lltype.malloc(rffi.CArray(rffi.FLOAT), size,
                                        flavor='raw', zero=True,
                                        track_allocation=False)
@@ -22,30 +31,33 @@
             threshold = 2
         return 1.0 / threshold   # the number is at most 0.5
 
-    def tick(self, hash, increment):
-        hash &= self.mask
-        counter = float(self.timetable[hash]) + increment
+    def get_index(self, hash):
+        """Return the index (< self.size) from a hash value.  This keeps
+        the *high* bits of hash!  Be sure that hash is computed correctly."""
+        return intmask(r_uint32(hash) >> self.shift)
+    get_index._always_inline_ = True
+
+    def tick(self, index, increment):
+        counter = float(self.timetable[index]) + increment
         if counter < 1.0:
-            self.timetable[hash] = r_singlefloat(counter)
+            self.timetable[index] = r_singlefloat(counter)
             return False
         else:
             return True
     tick._always_inline_ = True
 
-    def reset(self, hash):
-        hash &= self.mask
-        self.timetable[hash] = r_singlefloat(0.0)
+    def reset(self, index):
+        self.timetable[index] = r_singlefloat(0.0)
 
-    def lookup_chain(self, hash):
-        hash &= self.mask
-        return self.celltable[hash]
+    def lookup_chain(self, index):
+        return self.celltable[index]
 
-    def cleanup_chain(self, hash):
-        self.install_new_cell(hash, None)
+    def cleanup_chain(self, index):
+        self.reset(index)
+        self.install_new_cell(index, None)
 
-    def install_new_cell(self, hash, newcell):
-        hash &= self.mask
-        cell = self.celltable[hash]
+    def install_new_cell(self, index, newcell):
+        cell = self.celltable[index]
         keep = newcell
         while cell is not None:
             remove_me = cell.should_remove_jitcell()
@@ -54,7 +66,7 @@
                 cell.next = keep
                 keep = cell
             cell = nextcell
-        self.celltable[hash] = keep
+        self.celltable[index] = keep
 
     def set_decay(self, decay):
         """Set the decay, from 0 (none) to 1000 (max)."""
@@ -75,7 +87,7 @@
         # important in corner cases where we would suddenly compile more
         # than one loop because all counters reach the bound at the same
         # time, but where compiling all but the first one is pointless.
-        size = self.mask + 1
+        size = self.size
         pypy__decay_jit_counters(self.timetable, self.decay_by_mult, size)
 
 
diff --git a/rpython/jit/metainterp/test/test_counter.py b/rpython/jit/metainterp/test/test_counter.py
--- a/rpython/jit/metainterp/test/test_counter.py
+++ b/rpython/jit/metainterp/test/test_counter.py
@@ -1,21 +1,28 @@
 from rpython.jit.metainterp.counter import JitCounter
 
 
+def test_get_index():
+    jc = JitCounter(size=128)    # 7 bits
+    for i in range(10):
+        hash = 400000001 * i
+        index = jc.get_index(hash)
+        assert index == (hash >> (32 - 7))
+
 def test_tick():
     jc = JitCounter()
     incr = jc.compute_threshold(4)
     for i in range(5):
-        r = jc.tick(1234567, incr)
+        r = jc.tick(104, incr)
         assert r is (i >= 3)
     for i in range(5):
-        r = jc.tick(1234568, incr)
-        s = jc.tick(1234569, incr)
+        r = jc.tick(108, incr)
+        s = jc.tick(109, incr)
         assert r is (i >= 3)
         assert s is (i >= 3)
-    jc.reset(1234568)
+    jc.reset(108)
     for i in range(5):
-        r = jc.tick(1234568, incr)
-        s = jc.tick(1234569, incr)
+        r = jc.tick(108, incr)
+        s = jc.tick(109, incr)
         assert r is (i >= 3)
         assert s is True
 
@@ -30,21 +37,21 @@
             return False
     #
     jc = JitCounter()
-    assert jc.lookup_chain(1234567) is None
-    d1 = Dead()
-    jc.install_new_cell(1234567, d1)
-    assert jc.lookup_chain(1234567) is d1
+    assert jc.lookup_chain(104) is None
+    d1 = Dead() 
+    jc.install_new_cell(104, d1)
+    assert jc.lookup_chain(104) is d1
     d2 = Dead()
-    jc.install_new_cell(1234567, d2)
-    assert jc.lookup_chain(1234567) is d2
+    jc.install_new_cell(104, d2)
+    assert jc.lookup_chain(104) is d2
     assert d2.next is None
     #
     d3 = Alive()
-    jc.install_new_cell(1234567, d3)
-    assert jc.lookup_chain(1234567) is d3
+    jc.install_new_cell(104, d3)
+    assert jc.lookup_chain(104) is d3
     assert d3.next is None
     d4 = Alive()
-    jc.install_new_cell(1234567, d4)
-    assert jc.lookup_chain(1234567) is d3
+    jc.install_new_cell(104, d4)
+    assert jc.lookup_chain(104) is d3
     assert d3.next is d4
     assert d4.next is None
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -522,11 +522,6 @@
         #
         annhelper = MixLevelHelperAnnotator(self.translator.rtyper)
         for jd in self.jitdrivers_sd:
-            jd._set_jitcell_at_ptr = self._make_hook_graph(jd,
-                annhelper, jd.jitdriver.set_jitcell_at, annmodel.s_None,
-                s_BaseJitCell_not_None)
-            jd._get_jitcell_at_ptr = self._make_hook_graph(jd,
-                annhelper, jd.jitdriver.get_jitcell_at, s_BaseJitCell_or_None)
             jd._get_printable_location_ptr = self._make_hook_graph(jd,
                 annhelper, jd.jitdriver.get_printable_location, s_Str)
             jd._confirm_enter_jit_ptr = self._make_hook_graph(jd,
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -4,7 +4,7 @@
 from rpython.jit.codewriter import support, heaptracker, longlong
 from rpython.jit.metainterp import history
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
-from rpython.rlib.jit import PARAMETERS, BaseJitCell
+from rpython.rlib.jit import PARAMETERS
 from rpython.rlib.nonconst import NonConstant
 from rpython.rlib.objectmodel import specialize, we_are_translated, r_dict
 from rpython.rlib.rarithmetic import intmask
@@ -124,7 +124,7 @@
         return rffi.cast(lltype.Signed, x)
 
 
-class JitCell(BaseJitCell):
+class BaseJitCell(object):
     tracing = False
     dont_trace_here = chr(0)
     wref_procedure_token = None
@@ -267,7 +267,7 @@
         vinfo = jitdriver_sd.virtualizable_info
         index_of_virtualizable = jitdriver_sd.index_of_virtualizable
         num_green_args = jitdriver_sd.num_green_args
-        get_jitcell = self.make_jitcell_getter()
+        JitCell = self.make_jitcell_subclass()
         self.make_jitdriver_callbacks()
         confirm_enter_jit = self.confirm_enter_jit
         range_red_args = unrolling_iterable(
@@ -310,63 +310,63 @@
             #
             assert 0, "should have raised"
 
-        def bound_reached(cell, *args):
-            jitcounter.reset(
-            cell.counter = 0
+        def bound_reached(index, *args):
+            jitcounter.reset(index)
             if not confirm_enter_jit(*args):
                 return
             # start tracing
             from rpython.jit.metainterp.pyjitpl import MetaInterp
             metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
-            cell.tracing = True
-            cell.reset_counter()
+            greenargs = args[:num_green_args]
+            newcell = JitCell(*greenargs)
+            newcell.tracing = True
+            jitcounter.install_new_cell(index, newcell)
             try:
                 metainterp.compile_and_run_once(jitdriver_sd, *args)
             finally:
-                cell.tracing = False
-                cell.reset_counter()
+                newcell.tracing = False
 
-        def maybe_compile_and_run(threshold, *args):
+        def maybe_compile_and_run(increment_threshold, *args):
             """Entry point to the JIT.  Called at the point with the
             can_enter_jit() hint.
             """
-            # look for the cell corresponding to the current greenargs
+            # Look for the cell corresponding to the current greenargs.
+            # Search for the JitCell that is of the correct subclass of
+            # BaseJitCell, and that stores a key that compares equal
             greenargs = args[:num_green_args]
-            cell = get_jitcell(True, *greenargs)
-            mode = cell.mode
+            index = JitCell.get_index(*greenargs)
+            cell = jitcounter.lookup_chain(index)
+            while cell is not None:
+                if isinstance(cell, JitCell) and cell.comparekey(*greenargs):
+                    break    # found
+            else:
+                # not found. increment the counter
+                if jitcounter.tick(index, increment_threshold):
+                    bound_reached(index, *args)
+                return
 
-            if mode == MODE_COUNTING:
-                # update the profiling counter
-                n = cell.counter + threshold
-                if n <= self.THRESHOLD_LIMIT:       # bound not reached
-                    cell.counter = n
-                    return
-                else:
-                    bound_reached(cell, *args)
-                    return
-
-            else:
-                if mode != MODE_HAVE_PROC:
-                    assert mode == MODE_TRACING
-                    # tracing already happening in some outer invocation of
-                    # this function. don't trace a second time.
-                    return
-                if not confirm_enter_jit(*args):
-                    return
-                # machine code was already compiled for these greenargs
-                procedure_token = cell.get_procedure_token()
-                if procedure_token is None:   # it was a weakref that has been freed
-                    cell.counter = 0
-                    cell.mode = MODE_COUNTING
-                    return
-                # extract and unspecialize the red arguments to pass to
-                # the assembler
-                execute_args = ()
-                for i in range_red_args:
-                    execute_args += (unspecialize_value(args[i]), )
-                # run it!  this executes until interrupted by an exception
-                execute_assembler(procedure_token, *execute_args)
+            # Here, we have found 'cell'.
             #
+            if cell.tracing:
+                # tracing already happening in some outer invocation of
+                # this function. don't trace a second time.
+                return
+            # machine code was already compiled for these greenargs
+            procedure_token = cell.get_procedure_token()
+            if procedure_token is None:
+                # it was an aborted compilation, or maybe a weakref that
+                # has been freed
+                jitcounter.cleanup_chain(index)
+                return
+            if not confirm_enter_jit(*args):
+                return
+            # extract and unspecialize the red arguments to pass to
+            # the assembler
+            execute_args = ()
+            for i in range_red_args:
+                execute_args += (unspecialize_value(args[i]), )
+            # run it!  this executes until interrupted by an exception
+            execute_assembler(procedure_token, *execute_args)
             assert 0, "should not reach this point"
 
         maybe_compile_and_run._dont_inline_ = True
@@ -401,144 +401,45 @@
 
     # ----------
 
-    def make_jitcell_getter(self):
+    def make_jitcell_subclass(self):
         "NOT_RPYTHON"
-        if hasattr(self, 'jit_getter'):
-            return self.jit_getter
+        if hasattr(self, 'JitCell'):
+            return self.JitCell
         #
-        if self.jitdriver_sd._get_jitcell_at_ptr is None:
-            jit_getter = self._make_jitcell_getter_default()
-        else:
-            jit_getter = self._make_jitcell_getter_custom()
+        jitcounter = self.warmrunnerdesc.jitcounter
+        jitdriver_sd = self.jitdriver_sd
+        green_args_spec = unrolling_iterable([('g%d' % i, TYPE)
+                     for i, TYPE in enumerate(jitdriver_sd._green_args_spec)])
         #
-        unwrap_greenkey = self.make_unwrap_greenkey()
+        class JitCell(BaseJitCell):
+            def __init__(self, *greenargs):
+                i = 0
+                for attrname, _ in green_args_spec:
+                    setattr(self, attrname, greenargs[i])
+                    i = i + 1
+
+            def comparekey(self, *greenargs2):
+                i = 0
+                for attrname, TYPE in green_args_spec:
+                    item1 = getattr(self, attrname)
+                    if not equal_whatever(TYPE, item1, greenargs2[i]):
+                        return False
+                    i = i + 1
+                return True
+
+            @staticmethod
+            def get_index(*greenargs):
+                x = 0
+                i = 0
+                for TYPE in green_args_spec:
+                    item = greenargs[i]
+                    y = hash_whatever(TYPE, item)
+                    x = intmask((x ^ y) * 1405695061)  # prime number, 2**30~31
+                    i = i + 1
+                return jitcounter.get_index(x)
         #
-        def jit_cell_at_key(greenkey):
-            greenargs = unwrap_greenkey(greenkey)
-            return jit_getter(True, *greenargs)
-        self.jit_cell_at_key = jit_cell_at_key
-        self.jit_getter = jit_getter
-        #
-        return jit_getter
-
-    def _make_jitcell_getter_default(self):
-        "NOT_RPYTHON"
-        jitdriver_sd = self.jitdriver_sd
-        green_args_spec = unrolling_iterable(jitdriver_sd._green_args_spec)
-        #
-        def comparekey(greenargs1, greenargs2):
-            i = 0
-            for TYPE in green_args_spec:
-                if not equal_whatever(TYPE, greenargs1[i], greenargs2[i]):
-                    return False
-                i = i + 1
-            return True
-        #
-        def hashkey(greenargs):
-            x = 0x345678
-            i = 0
-            for TYPE in green_args_spec:
-                item = greenargs[i]
-                y = hash_whatever(TYPE, item)
-                x = intmask((1000003 * x) ^ y)
-                i = i + 1
-            return x
-        #
-        jitcell_dict = r_dict(comparekey, hashkey)
-        try:
-            self.warmrunnerdesc.stats.jitcell_dicts.append(jitcell_dict)
-        except AttributeError:
-            pass
-        #
-        def _cleanup_dict():
-            minimum = self.THRESHOLD_LIMIT // 20     # minimum 5%
-            killme = []
-            for key, cell in jitcell_dict.iteritems():
-                if cell.mode == MODE_COUNTING:
-                    cell.counter = int(cell.counter * 0.92)
-                    if cell.counter < minimum:
-                        killme.append(key)
-                elif (cell.mode == MODE_HAVE_PROC
-                      and cell.get_procedure_token() is None):
-                    killme.append(key)
-            for key in killme:
-                del jitcell_dict[key]
-        #
-        def _maybe_cleanup_dict():
-            # Once in a while, rarely, when too many entries have
-            # been put in the jitdict_dict, we do a cleanup phase:
-            # we decay all counters and kill entries with a too
-            # low counter.
-            self._trigger_automatic_cleanup += 1
-            if self._trigger_automatic_cleanup > 20000:
-                self._trigger_automatic_cleanup = 0
-                _cleanup_dict()
-        #
-        self._trigger_automatic_cleanup = 0
-        self._jitcell_dict = jitcell_dict       # for tests
-        #
-        def get_jitcell(build, *greenargs):
-            try:
-                cell = jitcell_dict[greenargs]
-            except KeyError:
-                if not build:
-                    return None
-                _maybe_cleanup_dict()
-                cell = JitCell()
-                jitcell_dict[greenargs] = cell
-            return cell
-        return get_jitcell
-
-    def _make_jitcell_getter_custom(self):
-        "NOT_RPYTHON"
-        rtyper = self.warmrunnerdesc.rtyper
-        get_jitcell_at_ptr = self.jitdriver_sd._get_jitcell_at_ptr
-        set_jitcell_at_ptr = self.jitdriver_sd._set_jitcell_at_ptr
-        lltohlhack = {}
-        # note that there is no equivalent of _maybe_cleanup_dict()
-        # in the case of custom getters.  We assume that the interpreter
-        # stores the JitCells on some objects that can go away by GC,
-        # like the PyCode objects in PyPy.
-        #
-        def get_jitcell(build, *greenargs):
-            fn = support.maybe_on_top_of_llinterp(rtyper, get_jitcell_at_ptr)
-            cellref = fn(*greenargs)
-            # <hacks>
-            if we_are_translated():
-                BASEJITCELL = lltype.typeOf(cellref)
-                cell = cast_base_ptr_to_instance(JitCell, cellref)
-            else:
-                if isinstance(cellref, (BaseJitCell, type(None))):
-                    BASEJITCELL = None
-                    cell = cellref
-                else:
-                    BASEJITCELL = lltype.typeOf(cellref)
-                    if cellref:
-                        cell = lltohlhack[rtyper.type_system.deref(cellref)]
-                    else:
-                        cell = None
-            if not build:
-                return cell
-            if cell is None:
-                cell = JitCell()
-                # <hacks>
-                if we_are_translated():
-                    cellref = cast_object_to_ptr(BASEJITCELL, cell)
-                else:
-                    if BASEJITCELL is None:
-                        cellref = cell
-                    else:
-                        if isinstance(BASEJITCELL, lltype.Ptr):
-                            cellref = lltype.malloc(BASEJITCELL.TO)
-                        else:
-                            assert False, "no clue"
-                        lltohlhack[rtyper.type_system.deref(cellref)] = cell
-                # </hacks>
-                fn = support.maybe_on_top_of_llinterp(rtyper,
-                                                      set_jitcell_at_ptr)
-                fn(cellref, *greenargs)
-            return cell
-        return get_jitcell
+        self.JitCell = JitCell
+        return JitCell
 
     # ----------
 
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -515,8 +515,8 @@
                                   if '.' not in name])
         self._heuristic_order = {}   # check if 'reds' and 'greens' are ordered
         self._make_extregistryentries()
-        self.get_jitcell_at = get_jitcell_at
-        self.set_jitcell_at = set_jitcell_at
+        assert get_jitcell_at is None, "get_jitcell_at no longer used"
+        assert set_jitcell_at is None, "set_jitcell_at no longer used"
         self.get_printable_location = get_printable_location
         self.confirm_enter_jit = confirm_enter_jit
         self.can_never_inline = can_never_inline
@@ -696,9 +696,6 @@
 #
 # Annotation and rtyping of some of the JitDriver methods
 
-class BaseJitCell(object):
-    __slots__ = ()
-
 
 class ExtEnterLeaveMarker(ExtRegistryEntry):
     # Replace a call to myjitdriver.jit_merge_point(**livevars)
@@ -746,10 +743,7 @@
 
     def annotate_hooks(self, **kwds_s):
         driver = self.instance.im_self
-        s_jitcell = self.bookkeeper.valueoftype(BaseJitCell)
         h = self.annotate_hook
-        h(driver.get_jitcell_at, driver.greens, **kwds_s)
-        h(driver.set_jitcell_at, driver.greens, [s_jitcell], **kwds_s)
         h(driver.get_printable_location, driver.greens, **kwds_s)
 
     def annotate_hook(self, func, variables, args_s=[], **kwds_s):

From noreply at buildbot.pypy.org  Wed Oct 30 18:17:54 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 30 Oct 2013 18:17:54 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: still in-progress
Message-ID: <20131030171754.34D831C1066@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67743:c5a737d0b87a
Date: 2013-10-30 18:17 +0100
http://bitbucket.org/pypy/pypy/changeset/c5a737d0b87a/

Log:	still in-progress

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2208,7 +2208,10 @@
             raise NotImplementedError(opname[opnum])
 
     def get_procedure_token(self, greenkey, with_compiled_targets=False):
-        cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey)
+        JitCell = self.jitdriver_sd.warmstate.JitCell
+        cell = JitCell.get_jit_cell_at_key(greenkey)
+        if cell is None:
+            return None
         token = cell.get_procedure_token()
         if with_compiled_targets:
             if not token:
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -28,8 +28,8 @@
 
     class FakeWarmRunnerState(object):
         def attach_procedure_to_interp(self, greenkey, procedure_token):
-            cell = self.jit_cell_at_key(greenkey)
-            cell.set_procedure_token(procedure_token)
+            assert greenkey == []
+            self._cell.set_procedure_token(procedure_token)
 
         def helper_func(self, FUNCPTR, func):
             from rpython.rtyper.annlowlevel import llhelper
@@ -38,9 +38,11 @@
         def get_location_str(self, args):
             return 'location'
 
-        def jit_cell_at_key(self, greenkey):
-            assert greenkey == []
-            return self._cell
+        class JitCell:
+            @staticmethod
+            def get_jit_cell_at_key(greenkey):
+                assert greenkey == []
+                return FakeWarmRunnerState._cell
         _cell = FakeJitCell()
 
         trace_limit = sys.maxint
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -512,12 +512,6 @@
         jd._maybe_compile_and_run_fn = maybe_compile_and_run
 
     def make_driverhook_graphs(self):
-        from rpython.rlib.jit import BaseJitCell
-        bk = self.rtyper.annotator.bookkeeper
-        classdef = bk.getuniqueclassdef(BaseJitCell)
-        s_BaseJitCell_or_None = annmodel.SomeInstance(classdef,
-                                                      can_be_None=True)
-        s_BaseJitCell_not_None = annmodel.SomeInstance(classdef)
         s_Str = annmodel.SomeString()
         #
         annhelper = MixLevelHelperAnnotator(self.translator.rtyper)
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -236,7 +236,7 @@
                 self.warmrunnerdesc.memory_manager.max_unroll_loops = value
 
     def disable_noninlinable_function(self, greenkey):
-        cell = self.jit_cell_at_key(greenkey)
+        cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
         cell.dont_trace_here = chr(20)
         debug_start("jit-disableinlining")
         loc = self.get_location_str(greenkey)
@@ -244,7 +244,7 @@
         debug_stop("jit-disableinlining")
 
     def attach_procedure_to_interp(self, greenkey, procedure_token):
-        cell = self.jit_cell_at_key(greenkey)
+        cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
         old_token = cell.get_procedure_token()
         cell.set_procedure_token(procedure_token)
         if old_token is not None:
@@ -332,7 +332,9 @@
             """
             # Look for the cell corresponding to the current greenargs.
             # Search for the JitCell that is of the correct subclass of
-            # BaseJitCell, and that stores a key that compares equal
+            # BaseJitCell, and that stores a key that compares equal.
+            # These few lines inline some logic that is also on the
+            # JitCell class, to avoid computing the hash several times.
             greenargs = args[:num_green_args]
             index = JitCell.get_index(*greenargs)
             cell = jitcounter.lookup_chain(index)
@@ -410,6 +412,7 @@
         jitdriver_sd = self.jitdriver_sd
         green_args_spec = unrolling_iterable([('g%d' % i, TYPE)
                      for i, TYPE in enumerate(jitdriver_sd._green_args_spec)])
+        unwrap_greenkey = self.make_unwrap_greenkey()
         #
         class JitCell(BaseJitCell):
             def __init__(self, *greenargs):
@@ -437,6 +440,34 @@
                     x = intmask((x ^ y) * 1405695061)  # prime number, 2**30~31
                     i = i + 1
                 return jitcounter.get_index(x)
+
+            @staticmethod
+            def get_jitcell(*greenargs):
+                index = JitCell.get_index(*greenargs)
+                cell = jitcounter.lookup_chain(index)
+                while cell is not None:
+                    if (isinstance(cell, JitCell) and
+                            cell.comparekey(*greenargs)):
+                        return cell
+                return None
+
+            @staticmethod
+            def get_jit_cell_at_key(greenkey):
+                greenargs = unwrap_greenkey(greenkey)
+                return JitCell.get_jitcell(*greenargs)
+
+            @staticmethod
+            def ensure_jit_cell_at_key(greenkey):
+                greenargs = unwrap_greenkey(greenkey)
+                index = JitCell.get_index(*greenargs)
+                cell = jitcounter.lookup_chain(index)
+                while cell is not None:
+                    if (isinstance(cell, JitCell) and
+                            cell.comparekey(*greenargs)):
+                        return cell
+                newcell = JitCell(*greenargs)
+                jitcounter.install_new_cell(index, newcell)
+                return newcell
         #
         self.JitCell = JitCell
         return JitCell
@@ -449,14 +480,14 @@
         #
         warmrunnerdesc = self.warmrunnerdesc
         unwrap_greenkey = self.make_unwrap_greenkey()
-        jit_getter = self.make_jitcell_getter()
+        JitCell = self.make_jitcell_subclass()
         jd = self.jitdriver_sd
         cpu = self.cpu
 
         def can_inline_greenargs(*greenargs):
             if can_never_inline(*greenargs):
                 return False
-            cell = jit_getter(False, *greenargs)
+            cell = JitCell.get_jitcell(*greenargs)
             if cell is not None and ord(cell.dont_trace_here) != 0:
                 return False
             return True
@@ -481,7 +512,7 @@
         redargtypes = ''.join([kind[0] for kind in jd.red_args_types])
 
         def get_assembler_token(greenkey):
-            cell = self.jit_cell_at_key(greenkey)
+            cell = self.ensure_jit_cell_at_key(greenkey)
             procedure_token = cell.get_procedure_token()
             if procedure_token is None:
                 from rpython.jit.metainterp.compile import compile_tmp_callback

From noreply at buildbot.pypy.org  Wed Oct 30 19:08:21 2013
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 30 Oct 2013 19:08:21 +0100 (CET)
Subject: [pypy-commit] pypy default: fix numpy module alias in tests
Message-ID: <20131030180821.043FC1C067F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r67744:5cb40ce76a36
Date: 2013-10-30 14:07 -0400
http://bitbucket.org/pypy/pypy/changeset/5cb40ce76a36/

Log:	fix numpy module alias in tests

diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -12,6 +12,7 @@
                 import numpy
             else:
                 from . import dummy_module as numpy
+                sys.modules['numpy'] = numpy
             sys.modules['numpypy'] = numpy
         else:
             import os

From noreply at buildbot.pypy.org  Wed Oct 30 19:12:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 30 Oct 2013 19:12:51 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: in-progress on the guard counters
	now
Message-ID: <20131030181251.EECDA1C067F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67745:22b67a551eb0
Date: 2013-10-30 19:12 +0100
http://bitbucket.org/pypy/pypy/changeset/22b67a551eb0/

Log:	in-progress on the guard counters now

diff --git a/rpython/jit/codewriter/longlong.py b/rpython/jit/codewriter/longlong.py
--- a/rpython/jit/codewriter/longlong.py
+++ b/rpython/jit/codewriter/longlong.py
@@ -25,6 +25,7 @@
     getfloatstorage = lambda x: x
     getrealfloat    = lambda x: x
     gethash         = compute_hash
+    gethash_fast    = longlong2float.float2longlong
     is_longlong     = lambda TYPE: False
 
     # -------------------------------------
@@ -40,6 +41,7 @@
     getfloatstorage = longlong2float.float2longlong
     getrealfloat    = longlong2float.longlong2float
     gethash         = lambda xll: rarithmetic.intmask(xll - (xll >> 32))
+    gethash_fast    = gethash
     is_longlong     = lambda TYPE: (TYPE is lltype.SignedLongLong or
                                     TYPE is lltype.UnsignedLongLong)
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -483,11 +483,7 @@
     pass
 
 class ResumeGuardDescr(ResumeDescr):
-    _counter = 0        # on a GUARD_VALUE, there is one counter per value;
-    _counters = None    # they get stored in _counters then.
-
     # this class also gets the following attributes stored by resume.py code
-
     # XXX move all of unused stuff to guard_op, now that we can have
     #     a separate class, so it does not survive that long
     rd_snapshot = None
@@ -498,18 +494,26 @@
     rd_virtuals = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
 
-    CNT_BASE_MASK  =  0x0FFFFFFF     # the base counter value
-    CNT_BUSY_FLAG  =  0x10000000     # if set, busy tracing from the guard
-    CNT_TYPE_MASK  =  0x60000000     # mask for the type
+    status = 0
 
-    CNT_INT        =  0x20000000
-    CNT_REF        =  0x40000000
-    CNT_FLOAT      =  0x60000000
+    ST_BUSY_FLAG    = 0x01     # if set, busy tracing from the guard
+    ST_TYPE_MASK    = 0x06     # mask for the type (TY_xxx)
+    ST_SHIFT        = 3        # in "status >> ST_SHIFT" is stored:
+                               # - if TY_NONE, the jitcounter index directly
+                               # - otherwise, the guard_value failarg index
+    TY_NONE         = 0x00
+    TY_INT          = 0x02
+    TY_REF          = 0x04
+    TY_FLOAT        = 0x06
 
-    def store_final_boxes(self, guard_op, boxes):
+    def store_final_boxes(self, guard_op, boxes, metainterp_sd):
         guard_op.setfailargs(boxes)
         self.rd_count = len(boxes)
         self.guard_opnum = guard_op.getopnum()
+        #
+        if metainterp_sd.warmrunnerdesc is not None:   # for tests
+            jitcounter = metainterp_sd.warmrunnerdesc.jitcounter
+            self.status = jitcounter.fetch_next_index() << self.ST_SHIFT
 
     def make_a_counter_per_value(self, guard_value_op):
         assert guard_value_op.getopnum() == rop.GUARD_VALUE
@@ -519,18 +523,15 @@
         except ValueError:
             return     # xxx probably very rare
         else:
-            if i > self.CNT_BASE_MASK:
-                return    # probably never, but better safe than sorry
             if box.type == history.INT:
-                cnt = self.CNT_INT
+                ty = self.TY_INT
             elif box.type == history.REF:
-                cnt = self.CNT_REF
+                ty = self.TY_REF
             elif box.type == history.FLOAT:
-                cnt = self.CNT_FLOAT
+                ty = self.TY_FLOAT
             else:
                 assert 0, box.type
-            assert cnt > self.CNT_BASE_MASK
-            self._counter = cnt | i
+            self.status = ty | (i << self.ST_SHIFT)
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         if self.must_compile(deadframe, metainterp_sd, jitdriver_sd):
@@ -557,65 +558,55 @@
     _trace_and_compile_from_bridge._dont_inline_ = True
 
     def must_compile(self, deadframe, metainterp_sd, jitdriver_sd):
-        trace_eagerness = jitdriver_sd.warmstate.trace_eagerness
+        jitcounter = metainterp_sd.warmrunnerdesc.jitcounter
         #
-        if self._counter <= self.CNT_BASE_MASK:
-            # simple case: just counting from 0 to trace_eagerness
-            self._counter += 1
-            return self._counter >= trace_eagerness
+        if self.status & (self.ST_BUSY_FLAG | self.ST_TYPE_MASK) == 0:
+            # common case: this is not a guard_value, and we are not
+            # already busy tracing.  The rest of self.status stores a
+            # valid per-guard index in the jitcounter.
+            index = self.status >> self.ST_SHIFT
         #
         # do we have the BUSY flag?  If so, we're tracing right now, e.g. in an
         # outer invocation of the same function, so don't trace again for now.
-        elif self._counter & self.CNT_BUSY_FLAG:
+        elif self.status & self.ST_BUSY_FLAG:
             return False
         #
-        else: # we have a GUARD_VALUE that fails.  Make a _counters instance
-            # (only now, when the guard is actually failing at least once),
-            # and use it to record some statistics about the failing values.
-            index = self._counter & self.CNT_BASE_MASK
-            typetag = self._counter & self.CNT_TYPE_MASK
-            counters = self._counters
-            if typetag == self.CNT_INT:
-                intvalue = metainterp_sd.cpu.get_int_value(
-                    deadframe, index)
-                if counters is None:
-                    self._counters = counters = ResumeGuardCountersInt()
-                else:
-                    assert isinstance(counters, ResumeGuardCountersInt)
-                counter = counters.see_int(intvalue)
+        else:    # we have a GUARD_VALUE that fails.
+            from rpython.rlib.objectmodel import current_object_addr_as_int
+
+            index = self.status >> self.ST_SHIFT
+            typetag = self.status & self.ST_TYPE_MASK
+
+            # fetch the actual value of the guard_value, possibly turning
+            # it to an integer
+            if typetag == self.TY_INT:
+                intval = metainterp_sd.cpu.get_int_value(deadframe, index)
             elif typetag == self.CNT_REF:
-                refvalue = metainterp_sd.cpu.get_ref_value(
-                    deadframe, index)
-                if counters is None:
-                    self._counters = counters = ResumeGuardCountersRef()
-                else:
-                    assert isinstance(counters, ResumeGuardCountersRef)
-                counter = counters.see_ref(refvalue)
+                refval = metainterp_sd.cpu.get_ref_value(deadframe, index)
+                intval = current_object_addr_as_int(refval)
             elif typetag == self.CNT_FLOAT:
-                floatvalue = metainterp_sd.cpu.get_float_value(
-                    deadframe, index)
-                if counters is None:
-                    self._counters = counters = ResumeGuardCountersFloat()
-                else:
-                    assert isinstance(counters, ResumeGuardCountersFloat)
-                counter = counters.see_float(floatvalue)
+                floatval = metainterp_sd.cpu.get_float_value(deadframe, index)
+                intval = longlong.gethash_fast(floatval)
             else:
                 assert 0, typetag
-            return counter >= trace_eagerness
+
+            hash = (current_object_addr_as_int(self) * 777767777 +
+                    intval * 1442968193)
+            index = jitcounter.get_index(hash)
+        #
+        increment = jitdriver_sd.warmstate.increment_trace_eagerness
+        return jitcounter.tick(index, increment)
 
     def start_compiling(self):
         # start tracing and compiling from this guard.
-        self._counter |= self.CNT_BUSY_FLAG
+        self.status |= self.ST_BUSY_FLAG
 
     def done_compiling(self):
-        # done tracing and compiling from this guard.  Either the bridge has
-        # been successfully compiled, in which case whatever value we store
-        # in self._counter will not be seen any more, or not, in which case
-        # we should reset the counter to 0, in order to wait a bit until the
-        # next attempt.
-        if self._counter >= 0:
-            self._counter = 0
-        self._counters = None
+        # done tracing and compiling from this guard.  Note that if the
+        # bridge has not been successfully compiled, the jitcounter for
+        # it was reset to 0 already by jitcounter.tick() and not
+        # incremented at all as long as ST_BUSY_FLAG was set.
+        self.status &= ~self.ST_BUSY_FLAG
 
     def compile_and_attach(self, metainterp, new_loop):
         # We managed to create a bridge.  Attach the new operations
@@ -745,69 +736,6 @@
         return res
 
 
-class AbstractResumeGuardCounters(object):
-    # Completely custom algorithm for now: keep 5 pairs (value, counter),
-    # and when we need more, we discard the middle pair (middle in the
-    # current value of the counter).  That way, we tend to keep the
-    # values with a high counter, but also we avoid always throwing away
-    # the most recently added value.  **THIS ALGO MUST GO AWAY AT SOME POINT**
-    pass
-
-def _see(self, newvalue):
-    # find and update an existing counter
-    unused = -1
-    for i in range(5):
-        cnt = self.counters[i]
-        if cnt:
-            if self.values[i] == newvalue:
-                cnt += 1
-                self.counters[i] = cnt
-                return cnt
-        else:
-            unused = i
-    # not found.  Use a previously unused entry, if there is one
-    if unused >= 0:
-        self.counters[unused] = 1
-        self.values[unused] = newvalue
-        return 1
-    # no unused entry.  Overwrite the middle one.  Computed with indices
-    # a, b, c meaning the highest, second highest, and third highest
-    # entries.
-    a = 0
-    b = c = -1
-    for i in range(1, 5):
-        if self.counters[i] > self.counters[a]:
-            c = b
-            b = a
-            a = i
-        elif b < 0 or self.counters[i] > self.counters[b]:
-            c = b
-            b = i
-        elif c < 0 or self.counters[i] > self.counters[c]:
-            c = i
-    self.counters[c] = 1
-    self.values[c] = newvalue
-    return 1
-
-class ResumeGuardCountersInt(AbstractResumeGuardCounters):
-    def __init__(self):
-        self.counters = [0] * 5
-        self.values = [0] * 5
-    see_int = func_with_new_name(_see, 'see_int')
-
-class ResumeGuardCountersRef(AbstractResumeGuardCounters):
-    def __init__(self):
-        self.counters = [0] * 5
-        self.values = [history.ConstPtr.value] * 5
-    see_ref = func_with_new_name(_see, 'see_ref')
-
-class ResumeGuardCountersFloat(AbstractResumeGuardCounters):
-    def __init__(self):
-        self.counters = [0] * 5
-        self.values = [longlong.ZEROF] * 5
-    see_float = func_with_new_name(_see, 'see_float')
-
-
 class ResumeFromInterpDescr(ResumeDescr):
     def __init__(self, original_greenkey):
         self.original_greenkey = original_greenkey
diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -22,6 +22,7 @@
                                        flavor='raw', zero=True,
                                        track_allocation=False)
         self.celltable = [None] * size
+        self._nextindex = 0
 
     def compute_threshold(self, threshold):
         """Return the 'increment' value corresponding to the given number."""
@@ -37,6 +38,11 @@
         return intmask(r_uint32(hash) >> self.shift)
     get_index._always_inline_ = True
 
+    def fetch_next_index(self):
+        result = self._nextindex
+        self._nextindex = (result + 1) & self.get_index(-1)
+        return result
+
     def tick(self, index, increment):
         counter = float(self.timetable[index]) + increment
         if counter < 1.0:
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -584,7 +584,7 @@
                 raise resume.TagOverflow
         except resume.TagOverflow:
             raise compile.giveup()
-        descr.store_final_boxes(op, newboxes)
+        descr.store_final_boxes(op, newboxes, self.metainterp_sd)
         #
         if op.getopnum() == rop.GUARD_VALUE:
             if self.getvalue(op.getarg(0)) in self.bool_boxes:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -312,7 +312,7 @@
     def __init__(self, metainterp_sd=None, original_greenkey=None):
         self.metainterp_sd = metainterp_sd
         self.original_greenkey = original_greenkey
-    def store_final_boxes(self, op, boxes):
+    def store_final_boxes(self, op, boxes, metainterp_sd):
         op.setfailargs(boxes)
     def __eq__(self, other):
         return type(self) is type(other)      # xxx obscure
diff --git a/rpython/jit/metainterp/test/test_counter.py b/rpython/jit/metainterp/test/test_counter.py
--- a/rpython/jit/metainterp/test/test_counter.py
+++ b/rpython/jit/metainterp/test/test_counter.py
@@ -8,6 +8,11 @@
         index = jc.get_index(hash)
         assert index == (hash >> (32 - 7))
 
+def test_fetch_next_index():
+    jc = JitCounter(size=4)
+    lst = [jc.fetch_next_index() for i in range(10)]
+    assert lst == [0, 1, 2, 3, 0, 1, 2, 3, 0, 1]
+
 def test_tick():
     jc = JitCounter()
     incr = jc.compute_threshold(4)

From noreply at buildbot.pypy.org  Wed Oct 30 19:14:02 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 30 Oct 2013 19:14:02 +0100 (CET)
Subject: [pypy-commit] pypy fix-trace-jit: random cleanup
Message-ID: <20131030181402.503451C067F@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: fix-trace-jit
Changeset: r67746:b8ceecb25294
Date: 2013-10-30 11:12 -0700
http://bitbucket.org/pypy/pypy/changeset/b8ceecb25294/

Log:	random cleanup

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -1,5 +1,5 @@
 =====================================
-PyPy: Python in Python Implementation 
+PyPy: Python in Python Implementation
 =====================================
 
 Welcome to PyPy!
@@ -26,9 +26,11 @@
 Building
 ========
 
-build with::
+build with:
 
-  rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+.. code-block:: console
+
+    $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
 
 This ends up with ``pypy-c`` binary in the main pypy directory. We suggest
 to use virtualenv with the resulting pypy-c as the interpreter, you can

From noreply at buildbot.pypy.org  Wed Oct 30 19:14:03 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 30 Oct 2013 19:14:03 +0100 (CET)
Subject: [pypy-commit] pypy fix-trace-jit: merged upstream
Message-ID: <20131030181403.7F3231C067F@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: fix-trace-jit
Changeset: r67747:ca3f8f1ebb22
Date: 2013-10-30 11:13 -0700
http://bitbucket.org/pypy/pypy/changeset/ca3f8f1ebb22/

Log:	merged upstream

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -272,7 +272,7 @@
 
     def _trace(self, frame, event, w_arg, operr=None):
         if self.is_tracing or frame.hide():
-            return True
+            return
 
         space = self.space
 
@@ -312,7 +312,7 @@
                     event == 'c_call' or
                     event == 'c_return' or
                     event == 'c_exception'):
-                return False
+                return
 
             last_exception = frame.last_exception
             if event == 'leaveframe':
@@ -332,7 +332,6 @@
             finally:
                 frame.last_exception = last_exception
                 self.is_tracing -= 1
-        return False
 
     def checksignals(self):
         """Similar to PyErr_CheckSignals().  If called in the main thread,

From noreply at buildbot.pypy.org  Wed Oct 30 19:48:07 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 30 Oct 2013 19:48:07 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Fixes
Message-ID: <20131030184807.523B01C0330@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67748:3f10fdb46ecf
Date: 2013-10-30 19:26 +0100
http://bitbucket.org/pypy/pypy/changeset/3f10fdb46ecf/

Log:	Fixes

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -581,10 +581,10 @@
             # it to an integer
             if typetag == self.TY_INT:
                 intval = metainterp_sd.cpu.get_int_value(deadframe, index)
-            elif typetag == self.CNT_REF:
+            elif typetag == self.TY_REF:
                 refval = metainterp_sd.cpu.get_ref_value(deadframe, index)
                 intval = current_object_addr_as_int(refval)
-            elif typetag == self.CNT_FLOAT:
+            elif typetag == self.TY_FLOAT:
                 floatval = metainterp_sd.cpu.get_float_value(deadframe, index)
                 intval = longlong.gethash_fast(floatval)
             else:
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -512,7 +512,7 @@
         redargtypes = ''.join([kind[0] for kind in jd.red_args_types])
 
         def get_assembler_token(greenkey):
-            cell = self.ensure_jit_cell_at_key(greenkey)
+            cell = JitCell.ensure_jit_cell_at_key(greenkey)
             procedure_token = cell.get_procedure_token()
             if procedure_token is None:
                 from rpython.jit.metainterp.compile import compile_tmp_callback

From noreply at buildbot.pypy.org  Wed Oct 30 19:48:08 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 30 Oct 2013 19:48:08 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Might help in the presence of many
	jitdrivers.
Message-ID: <20131030184808.843731C0330@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67749:13fbd165c395
Date: 2013-10-30 19:47 +0100
http://bitbucket.org/pypy/pypy/changeset/13fbd165c395/

Log:	Might help in the presence of many jitdrivers.

diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -413,6 +413,7 @@
         green_args_spec = unrolling_iterable([('g%d' % i, TYPE)
                      for i, TYPE in enumerate(jitdriver_sd._green_args_spec)])
         unwrap_greenkey = self.make_unwrap_greenkey()
+        random_initial_value = hash(self)
         #
         class JitCell(BaseJitCell):
             def __init__(self, *greenargs):
@@ -432,7 +433,7 @@
 
             @staticmethod
             def get_index(*greenargs):
-                x = 0
+                x = random_initial_value
                 i = 0
                 for TYPE in green_args_spec:
                     item = greenargs[i]

From noreply at buildbot.pypy.org  Wed Oct 30 19:48:09 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 30 Oct 2013 19:48:09 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Fix for tests
Message-ID: <20131030184809.9DEF31C0330@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67750:7ab619834c25
Date: 2013-10-30 19:47 +0100
http://bitbucket.org/pypy/pypy/changeset/7ab619834c25/

Log:	Fix for tests

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -583,7 +583,7 @@
                 intval = metainterp_sd.cpu.get_int_value(deadframe, index)
             elif typetag == self.TY_REF:
                 refval = metainterp_sd.cpu.get_ref_value(deadframe, index)
-                intval = current_object_addr_as_int(refval)
+                intval = lltype.cast_ptr_to_int(refval)
             elif typetag == self.TY_FLOAT:
                 floatval = metainterp_sd.cpu.get_float_value(deadframe, index)
                 intval = longlong.gethash_fast(floatval)
diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -30,7 +30,7 @@
             return 0.0   # no increment, never reach 1.0
         if threshold < 2:
             threshold = 2
-        return 1.0 / threshold   # the number is at most 0.5
+        return 1.0 / (threshold - 0.001)   # the number is at most 0.500xx
 
     def get_index(self, hash):
         """Return the index (< self.size) from a hash value.  This keeps
@@ -110,3 +110,21 @@
 pypy__decay_jit_counters = rffi.llexternal(
     "pypy__decay_jit_counters", [rffi.FLOATP, lltype.Float, lltype.Signed],
     lltype.Void, compilation_info=eci, _nowrapper=True, sandboxsafe=True)
+
+
+# ____________________________________________________________
+#
+# A non-RPython version that avoids issues with rare random collisions,
+# which make all tests brittle
+
+class DeterministicJitCounter(JitCounter):
+    def __init__(self):
+        from collections import defaultdict
+        JitCounter.__init__(self, size=8)
+        zero = r_singlefloat(0.0)
+        self.timetable = defaultdict(lambda: zero)
+        self.celltable = defaultdict(lambda: None)
+
+    def get_index(self, hash):
+        "NOT_RPYTHON"
+        return hash
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -205,8 +205,11 @@
         vrefinfo = VirtualRefInfo(self)
         self.codewriter.setup_vrefinfo(vrefinfo)
         #
-        from rpython.jit.metainterp.counter import JitCounter
-        self.jitcounter = JitCounter()
+        from rpython.jit.metainterp import counter
+        if self.cpu.translate_support_code:
+            self.jitcounter = counter.JitCounter()
+        else:
+            self.jitcounter = counter.DeterministicJitCounter()
         #
         self.hooks = policy.jithookiface
         self.make_virtualizable_infos()

From noreply at buildbot.pypy.org  Wed Oct 30 23:05:18 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 30 Oct 2013 23:05:18 +0100 (CET)
Subject: [pypy-commit] pypy fix-trace-jit: Bump the trace limit when enable
	a sys.settrace()
Message-ID: <20131030220518.106441C029A@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: fix-trace-jit
Changeset: r67751:6a295fa19a43
Date: 2013-10-30 15:04 -0700
http://bitbucket.org/pypy/pypy/changeset/6a295fa19a43/

Log:	Bump the trace limit when enable a sys.settrace()

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -224,6 +224,7 @@
         else:
             self.force_all_frames()
             self.w_tracefunc = w_func
+            jit.set_param(None, 'trace_limit', 10000)
 
     def gettrace(self):
         return jit.promote(self.w_tracefunc)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -455,7 +455,7 @@
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
               'function_threshold': 1619, # slightly more than one above, also prime
               'trace_eagerness': 200,
-              'trace_limit': 8000,
+              'trace_limit': 6000,
               'inlining': 1,
               'loop_longevity': 1000,
               'retrace_limit': 5,

From noreply at buildbot.pypy.org  Wed Oct 30 23:08:46 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 30 Oct 2013 23:08:46 +0100 (CET)
Subject: [pypy-commit] pypy fix-trace-jit: Added a comment
Message-ID: <20131030220846.CB1601C029A@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: fix-trace-jit
Changeset: r67752:7629b9da95c3
Date: 2013-10-30 15:07 -0700
http://bitbucket.org/pypy/pypy/changeset/7629b9da95c3/

Log:	Added a comment

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -224,6 +224,8 @@
         else:
             self.force_all_frames()
             self.w_tracefunc = w_func
+            # Increase the JIT's trace_limit when we have a tracefunc, it
+            # generates a ton of extra ops.
             jit.set_param(None, 'trace_limit', 10000)
 
     def gettrace(self):

From noreply at buildbot.pypy.org  Wed Oct 30 23:08:48 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 30 Oct 2013 23:08:48 +0100 (CET)
Subject: [pypy-commit] pypy fix-trace-jit: Close branch for merge
Message-ID: <20131030220848.192971C029A@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: fix-trace-jit
Changeset: r67753:7a3ca5d98ff0
Date: 2013-10-30 15:07 -0700
http://bitbucket.org/pypy/pypy/changeset/7a3ca5d98ff0/

Log:	Close branch for merge


From noreply at buildbot.pypy.org  Wed Oct 30 23:08:49 2013
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 30 Oct 2013 23:08:49 +0100 (CET)
Subject: [pypy-commit] pypy default: Fixed sys.settrace() of the pypy
 interpreter when under the JIT.
Message-ID: <20131030220849.6F6241C029A@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r67754:2a993c608319
Date: 2013-10-30 15:08 -0700
http://bitbucket.org/pypy/pypy/changeset/2a993c608319/

Log:	Fixed sys.settrace() of the pypy interpreter when under the JIT.
	Properly run the tracefunction at every bytecode.

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -1,5 +1,5 @@
 =====================================
-PyPy: Python in Python Implementation 
+PyPy: Python in Python Implementation
 =====================================
 
 Welcome to PyPy!
@@ -26,9 +26,11 @@
 Building
 ========
 
-build with::
+build with:
 
-  rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+.. code-block:: console
+
+    $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
 
 This ends up with ``pypy-c`` binary in the main pypy directory. We suggest
 to use virtualenv with the resulting pypy-c as the interpreter, you can
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -10,7 +10,7 @@
 from rpython.rlib.rarithmetic import r_uint
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
-    UserDelAction, FrameTraceAction)
+    UserDelAction)
 from pypy.interpreter.error import (OperationError, operationerrfmt,
     new_exception_class)
 from pypy.interpreter.argument import Arguments
@@ -330,7 +330,6 @@
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
-        self.frame_trace_action = FrameTraceAction(self)
         self._code_of_sys_exc_info = None
 
         from pypy.interpreter.pycode import cpython_magic, default_magic
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -27,13 +27,10 @@
     def __init__(self, space):
         self.space = space
         self.topframeref = jit.vref_None
-        # tracing: space.frame_trace_action.fire() must be called to ensure
-        # that tracing occurs whenever self.w_tracefunc or self.is_tracing
-        # is modified.
-        self.w_tracefunc = None        # if not None, no JIT
+        self.w_tracefunc = None
         self.is_tracing = 0
         self.compiler = space.createcompiler()
-        self.profilefunc = None        # if not None, no JIT
+        self.profilefunc = None
         self.w_profilefuncarg = None
 
     def gettopframe(self):
@@ -76,9 +73,6 @@
                 frame_vref()
             jit.virtual_ref_finish(frame_vref, frame)
 
-        if self.gettrace() is not None and not frame.hide():
-            self.space.frame_trace_action.fire()
-
     # ________________________________________________________________
 
     def c_call_trace(self, frame, w_func, args=None):
@@ -123,25 +117,77 @@
     def return_trace(self, frame, w_retval):
         "Trace the return from a function"
         if self.gettrace() is not None:
-            return_from_hidden = self._trace(frame, 'return', w_retval)
-            # special case: if we are returning from a hidden function,
-            # then maybe we have to fire() the action again; otherwise
-            # it will not be called.  See test_trace_hidden_prints.
-            if return_from_hidden:
-                self.space.frame_trace_action.fire()
+            self._trace(frame, 'return', w_retval)
 
     def bytecode_trace(self, frame, decr_by=TICK_COUNTER_STEP):
         "Trace function called before each bytecode."
         # this is split into a fast path and a slower path that is
         # not invoked every time bytecode_trace() is.
+        self.bytecode_only_trace(frame)
         actionflag = self.space.actionflag
         if actionflag.decrement_ticker(decr_by) < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
     bytecode_trace._always_inline_ = True
 
+    def bytecode_only_trace(self, frame):
+        """
+        Like bytecode_trace() but doesn't invoke any other events besides the
+        trace function.
+        """
+        if (frame.w_f_trace is None or self.is_tracing or
+            self.gettrace() is None):
+            return
+        self.run_trace_func(frame)
+    bytecode_only_trace._always_inline_ = True
+
+    @jit.unroll_safe
+    def run_trace_func(self, frame):
+        code = frame.pycode
+        if frame.instr_lb <= frame.last_instr < frame.instr_ub:
+            if frame.last_instr < frame.instr_prev_plus_one:
+                # We jumped backwards in the same line.
+                self._trace(frame, 'line', self.space.w_None)
+        else:
+            size = len(code.co_lnotab) / 2
+            addr = 0
+            line = code.co_firstlineno
+            p = 0
+            lineno = code.co_lnotab
+            while size > 0:
+                c = ord(lineno[p])
+                if (addr + c) > frame.last_instr:
+                    break
+                addr += c
+                if c:
+                    frame.instr_lb = addr
+
+                line += ord(lineno[p + 1])
+                p += 2
+                size -= 1
+
+            if size > 0:
+                while True:
+                    size -= 1
+                    if size < 0:
+                        break
+                    addr += ord(lineno[p])
+                    if ord(lineno[p + 1]):
+                        break
+                    p += 2
+                frame.instr_ub = addr
+            else:
+                frame.instr_ub = sys.maxint
+
+            if frame.instr_lb == frame.last_instr: # At start of line!
+                frame.f_lineno = line
+                self._trace(frame, 'line', self.space.w_None)
+
+        frame.instr_prev_plus_one = frame.last_instr + 1
+
     def bytecode_trace_after_exception(self, frame):
         "Like bytecode_trace(), but without increasing the ticker."
         actionflag = self.space.actionflag
+        self.bytecode_only_trace(frame)
         if actionflag.get_ticker() < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
     bytecode_trace_after_exception._always_inline_ = 'try'
@@ -178,7 +224,9 @@
         else:
             self.force_all_frames()
             self.w_tracefunc = w_func
-            self.space.frame_trace_action.fire()
+            # Increase the JIT's trace_limit when we have a tracefunc, it
+            # generates a ton of extra ops.
+            jit.set_param(None, 'trace_limit', 10000)
 
     def gettrace(self):
         return jit.promote(self.w_tracefunc)
@@ -221,14 +269,13 @@
         is_tracing = self.is_tracing
         self.is_tracing = 0
         try:
-            self.space.frame_trace_action.fire()
             return self.space.call(w_func, w_args)
         finally:
             self.is_tracing = is_tracing
 
     def _trace(self, frame, event, w_arg, operr=None):
         if self.is_tracing or frame.hide():
-            return True
+            return
 
         space = self.space
 
@@ -260,7 +307,6 @@
             finally:
                 self.is_tracing -= 1
                 frame.locals2fast()
-                space.frame_trace_action.fire()
 
         # Profile cases
         if self.profilefunc is not None:
@@ -269,7 +315,7 @@
                     event == 'c_call' or
                     event == 'c_return' or
                     event == 'c_exception'):
-                return False
+                return
 
             last_exception = frame.last_exception
             if event == 'leaveframe':
@@ -289,7 +335,6 @@
             finally:
                 frame.last_exception = last_exception
                 self.is_tracing -= 1
-        return False
 
     def checksignals(self):
         """Similar to PyErr_CheckSignals().  If called in the main thread,
@@ -475,54 +520,3 @@
             except OperationError, e:
                 e.write_unraisable(space, descrname, w_obj)
                 e.clear(space)   # break up reference cycles
-
-class FrameTraceAction(AsyncAction):
-    """An action that calls the local trace functions (w_f_trace)."""
-
-    @jit.unroll_safe
-    def perform(self, executioncontext, frame):
-        if (frame.w_f_trace is None or executioncontext.is_tracing or
-            executioncontext.gettrace() is None):
-            return
-        code = frame.pycode
-        if frame.instr_lb <= frame.last_instr < frame.instr_ub:
-            if frame.last_instr < frame.instr_prev_plus_one:
-                # We jumped backwards in the same line.
-                executioncontext._trace(frame, 'line', self.space.w_None)
-        else:
-            size = len(code.co_lnotab) / 2
-            addr = 0
-            line = code.co_firstlineno
-            p = 0
-            lineno = code.co_lnotab
-            while size > 0:
-                c = ord(lineno[p])
-                if (addr + c) > frame.last_instr:
-                    break
-                addr += c
-                if c:
-                    frame.instr_lb = addr
-
-                line += ord(lineno[p + 1])
-                p += 2
-                size -= 1
-
-            if size > 0:
-                while True:
-                    size -= 1
-                    if size < 0:
-                        break
-                    addr += ord(lineno[p])
-                    if ord(lineno[p + 1]):
-                        break
-                    p += 2
-                frame.instr_ub = addr
-            else:
-                frame.instr_ub = sys.maxint
-
-            if frame.instr_lb == frame.last_instr: # At start of line!
-                frame.f_lineno = line
-                executioncontext._trace(frame, 'line', self.space.w_None)
-
-        frame.instr_prev_plus_one = frame.last_instr + 1
-        self.space.frame_trace_action.fire()     # continue tracing
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -464,8 +464,6 @@
         new_frame.instr_prev_plus_one = space.int_w(w_instr_prev_plus_one)
 
         self._setcellvars(cellvars)
-        # XXX what if the frame is in another thread??
-        space.frame_trace_action.fire()
 
     def hide(self):
         return self.pycode.hidden_applevel
@@ -759,7 +757,6 @@
         else:
             self.w_f_trace = w_trace
             self.f_lineno = self.get_last_lineno()
-            space.frame_trace_action.fire()
 
     def fdel_f_trace(self, space):
         self.w_f_trace = None
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -144,9 +144,11 @@
     def dispatch_bytecode(self, co_code, next_instr, ec):
         while True:
             self.last_instr = intmask(next_instr)
-            if not jit.we_are_jitted():
+            if jit.we_are_jitted():
+                ec.bytecode_only_trace(self)
+            else:
                 ec.bytecode_trace(self)
-                next_instr = r_uint(self.last_instr)
+            next_instr = r_uint(self.last_instr)
             opcode = ord(co_code[next_instr])
             next_instr += 1
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -425,6 +425,7 @@
         loop, = log.loops_by_id('call', is_entry_bridge=True)
         assert loop.match("""
             guard_value(i4, 1, descr=...)
+            guard_isnull(p5, descr=...)
             guard_nonnull_class(p12, ConstClass(W_IntObject), descr=...)
             guard_value(i8, 0, descr=...)
             guard_value(p2, ConstPtr(ptr21), descr=...)

From noreply at buildbot.pypy.org  Thu Oct 31 09:24:14 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 09:24:14 +0100 (CET)
Subject: [pypy-commit] pypy default: Translation fix: jit.set_param() is not
 supported with None if
Message-ID: <20131031082414.4653D1C2FF1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67760:00579deb9c76
Date: 2013-10-31 09:23 +0100
http://bitbucket.org/pypy/pypy/changeset/00579deb9c76/

Log:	Translation fix: jit.set_param() is not supported with None if seen
	itself by the jit (for now).

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -217,6 +217,7 @@
         if frame:     # else, the exception goes nowhere and is lost
             frame.last_exception = operror
 
+    @jit.dont_look_inside
     def settrace(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):

From noreply at buildbot.pypy.org  Thu Oct 31 10:27:44 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 10:27:44 +0100 (CET)
Subject: [pypy-commit] pypy default: Tweak 'more_objects_to_trace' to avoid
 recording *all* surviving young
Message-ID: <20131031092744.71C851C0F88@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67761:e2cc638553ad
Date: 2013-10-31 09:18 +0100
http://bitbucket.org/pypy/pypy/changeset/e2cc638553ad/

Log:	Tweak 'more_objects_to_trace' to avoid recording *all* surviving
	young objects during the marking phase.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1390,6 +1390,15 @@
         if self.young_rawmalloced_objects:
             self.remove_young_arrays_from_old_objects_pointing_to_young()
         #
+        # A special step in the STATE_MARKING phase.
+        if self.gc_state == STATE_MARKING:
+            # Copy the 'old_objects_pointing_to_young' list so far to
+            # 'more_objects_to_trace'.  Turn black objects back to gray.
+            # This is because these are precisely the old objects that
+            # have been modified and need rescanning.
+            self.old_objects_pointing_to_young.foreach(
+                self._add_to_more_objects_to_trace, None)
+        #
         # First, find the roots that point to young objects.  All nursery
         # objects found are copied out of the nursery, and the occasional
         # young raw-malloced object is flagged with GCFLAG_VISITED_RMY.
@@ -1459,10 +1468,14 @@
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
         debug_start("gc-minor-walkroots")
+        if self.gc_state == STATE_MARKING:
+            callback = IncrementalMiniMarkGC._trace_drag_out1_marking_phase
+        else:
+            callback = IncrementalMiniMarkGC._trace_drag_out1
         self.root_walker.walk_roots(
-            IncrementalMiniMarkGC._trace_drag_out1,  # stack roots
-            IncrementalMiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
-            None)                         # static in prebuilt gc
+            callback,     # stack roots
+            callback,     # static in prebuilt non-gc
+            None)         # static in prebuilt gc
         debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
@@ -1522,21 +1535,14 @@
                     interval_start = next_byte_start
                 #
                 # If we're incrementally marking right now, sorry, we also
-                # need to add the object to 'objects_to_trace' and have it
-                # fully traced very soon.
+                # need to add the object to 'more_objects_to_trace' and have
+                # it fully traced once at the end of the current marking phase.
                 if self.gc_state == STATE_MARKING:
                     self.header(obj).tid &= ~GCFLAG_VISITED
                     self.more_objects_to_trace.append(obj)
 
 
     def collect_oldrefs_to_nursery(self):
-        if self.gc_state == STATE_MARKING:
-            self._collect_oldrefs_to_nursery(True)
-        else:
-            self._collect_oldrefs_to_nursery(False)
-
-    @specialize.arg(1)
-    def _collect_oldrefs_to_nursery(self, state_is_marking):
         # Follow the old_objects_pointing_to_young list and move the
         # young objects they point to out of the nursery.
         oldlist = self.old_objects_pointing_to_young
@@ -1553,15 +1559,6 @@
             # have this flag set after a nursery collection.
             self.header(obj).tid |= GCFLAG_TRACK_YOUNG_PTRS
             #
-            # If the incremental major collection is currently at
-            # STATE_MARKING, then we must add to 'objects_to_trace' all
-            # objects that go through 'old_objects_pointing_to_young'.
-            # This basically turns black objects gray again, but also
-            # makes sure that we see otherwise-white objects.
-            if state_is_marking:
-                self.header(obj).tid &= ~GCFLAG_VISITED
-                self.more_objects_to_trace.append(obj)
-            #
             # Trace the 'obj' to replace pointers to nursery with pointers
             # outside the nursery, possibly forcing nursery objects out
             # and adding them to 'old_objects_pointing_to_young' as well.
@@ -1584,15 +1581,21 @@
 
 
     def _trace_drag_out1(self, root):
-        # In the MARKING state, we must also record this old object,
-        # if it is not VISITED yet.
-        if self.gc_state == STATE_MARKING:
-            obj = root.address[0]
-            if not self.is_in_nursery(obj):
-                if not self.header(obj).tid & GCFLAG_VISITED:
-                    self.more_objects_to_trace.append(obj)
+        self._trace_drag_out(root, None)
+
+    def _trace_drag_out1_marking_phase(self, root):
+        self._trace_drag_out(root, None)
         #
-        self._trace_drag_out(root, None)
+        # We are in the MARKING state: we must also record this object
+        # if it was young.  Don't bother with old objects in general,
+        # as they are anyway added to 'more_objects_to_trace' if they
+        # are modified (see _add_to_more_objects_to_trace).  But we do
+        # need to record the not-visited-yet (white) old objects.  So
+        # as a conservative approximation, we need to add the object to
+        # the list if and only if it doesn't have GCFLAG_VISITED yet.
+        obj = root.address[0]
+        if not self.header(obj).tid & GCFLAG_VISITED:
+            self.more_objects_to_trace.append(obj)
 
     def _trace_drag_out(self, root, ignored):
         obj = root.address[0]
@@ -1668,12 +1671,6 @@
         if self.has_gcptr(typeid):
             # we only have to do it if we have any gcptrs
             self.old_objects_pointing_to_young.append(newobj)
-        else:
-            # we don't need to add this to 'old_objects_pointing_to_young',
-            # but in the STATE_MARKING phase we still need this bit...
-            if self.gc_state == STATE_MARKING:
-                self.header(newobj).tid &= ~GCFLAG_VISITED
-                self.more_objects_to_trace.append(newobj)
 
     _trace_drag_out._always_inline_ = True
 
@@ -1756,6 +1753,10 @@
             old.append(new.pop())
         new.delete()
 
+    def _add_to_more_objects_to_trace(self, obj, ignored):
+        self.header(obj).tid &= ~GCFLAG_VISITED
+        self.more_objects_to_trace.append(obj)
+
     def minor_and_major_collection(self):
         # First, finish the current major gc, if there is one in progress.
         # This is a no-op if the gc_state is already STATE_SCANNING.

From noreply at buildbot.pypy.org  Thu Oct 31 10:27:46 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 10:27:46 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20131031092746.AA1461C0F88@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67762:7bb76620cb33
Date: 2013-10-31 09:23 +0100
http://bitbucket.org/pypy/pypy/changeset/7bb76620cb33/

Log:	merge heads

diff too long, truncating to 2000 out of 7781 lines

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -1,5 +1,5 @@
 =====================================
-PyPy: Python in Python Implementation 
+PyPy: Python in Python Implementation
 =====================================
 
 Welcome to PyPy!
@@ -26,9 +26,11 @@
 Building
 ========
 
-build with::
+build with:
 
-  rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+.. code-block:: console
+
+    $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
 
 This ends up with ``pypy-c`` binary in the main pypy directory. We suggest
 to use virtualenv with the resulting pypy-c as the interpreter, you can
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from . import core
-from .core import *
-from . import lib
-from .lib import *
-
-from __builtin__ import bool, int, long, float, complex, object, unicode, str
-
-from .core import round, abs, max, min
-
-__version__ = '1.7.0'
-
-__all__ = ['__version__']
-__all__ += core.__all__
-__all__ += lib.__all__
-
-#import sys
-#sys.modules.setdefault('numpy', sys.modules['numpypy'])
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from __future__ import division, absolute_import, print_function
-
-from . import multiarray
-from . import umath
-from . import numeric
-from .numeric import *
-from . import fromnumeric
-from .fromnumeric import *
-from . import shape_base
-from .shape_base import *
-
-from .fromnumeric import amax as max, amin as min, \
-    round_ as round
-from .numeric import absolute as abs
-
-__all__ = []
-__all__ += numeric.__all__
-__all__ += fromnumeric.__all__
-__all__ += shape_base.__all__
diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/_methods.py
+++ /dev/null
@@ -1,124 +0,0 @@
-"""
-Array methods which are called by the both the C-code for the method
-and the Python code for the NumPy-namespace function
-
-"""
-from __future__ import division, absolute_import, print_function
-
-import warnings
-
-from . import multiarray as mu
-from . import umath as um
-from .numeric import asanyarray
-from . import numerictypes as nt
-
-def _amax(a, axis=None, out=None, keepdims=False):
-    return um.maximum.reduce(a, axis=axis,
-                            out=out, keepdims=keepdims)
-
-def _amin(a, axis=None, out=None, keepdims=False):
-    return um.minimum.reduce(a, axis=axis,
-                            out=out, keepdims=keepdims)
-
-def _sum(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.add.reduce(a, axis=axis, dtype=dtype,
-                            out=out, keepdims=keepdims)
-
-def _prod(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.multiply.reduce(a, axis=axis, dtype=dtype,
-                            out=out, keepdims=keepdims)
-
-def _any(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.logical_or.reduce(a, axis=axis, dtype=dtype, out=out,
-                                keepdims=keepdims)
-
-def _all(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.logical_and.reduce(a, axis=axis, dtype=dtype, out=out,
-                                 keepdims=keepdims)
-
-def _count_reduce_items(arr, axis):
-    if axis is None:
-        axis = tuple(range(arr.ndim))
-    if not isinstance(axis, tuple):
-        axis = (axis,)
-    items = 1
-    for ax in axis:
-        items *= arr.shape[ax]
-    return items
-
-def _mean(a, axis=None, dtype=None, out=None, keepdims=False):
-    arr = asanyarray(a)
-
-    rcount = _count_reduce_items(arr, axis)
-    # Make this warning show up first
-    if rcount == 0:
-        warnings.warn("Mean of empty slice.", RuntimeWarning)
-
-
-    # Cast bool, unsigned int, and int to float64 by default
-    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
-        dtype = mu.dtype('f8')
-
-    ret = um.add.reduce(arr, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
-    if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(
-                ret, rcount, out=ret, casting='unsafe', subok=False)
-    else:
-        ret = ret.dtype.type(ret / rcount)
-
-    return ret
-
-def _var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
-    arr = asanyarray(a)
-
-    rcount = _count_reduce_items(arr, axis)
-    # Make this warning show up on top.
-    if ddof >= rcount:
-        warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning)
-
-    # Cast bool, unsigned int, and int to float64 by default
-    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
-        dtype = mu.dtype('f8')
-
-    # Compute the mean.
-    # Note that if dtype is not of inexact type then arraymean will
-    # not be either.
-    arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True)
-    if isinstance(arrmean, mu.ndarray):
-        arrmean = um.true_divide(
-                arrmean, rcount, out=arrmean, casting='unsafe', subok=False)
-    else:
-        arrmean = arrmean.dtype.type(arrmean / rcount)
-
-    # Compute sum of squared deviations from mean
-    # Note that x may not be inexact and that we need it to be an array,
-    # not a scalar.
-    x = asanyarray(arr - arrmean)
-    if issubclass(arr.dtype.type, nt.complexfloating):
-        x = um.multiply(x, um.conjugate(x), out=x).real
-    else:
-        x = um.multiply(x, x, out=x)
-    ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
-
-    # Compute degrees of freedom and make sure it is not negative.
-    rcount = max([rcount - ddof, 0])
-
-    # divide by degrees of freedom
-    if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(
-                ret, rcount, out=ret, casting='unsafe', subok=False)
-    else:
-        ret = ret.dtype.type(ret / rcount)
-
-    return ret
-
-def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
-    ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
-               keepdims=keepdims)
-
-    if isinstance(ret, mu.ndarray):
-        ret = um.sqrt(ret, out=ret)
-    else:
-        ret = ret.dtype.type(um.sqrt(ret))
-
-    return ret
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ /dev/null
@@ -1,751 +0,0 @@
-"""Array printing function
-
-$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $
-"""
-__all__ = ["array2string", "set_printoptions", "get_printoptions"]
-__docformat__ = 'restructuredtext'
-
-#
-# Written by Konrad Hinsen <hinsenk at ere.umontreal.ca>
-# last revision: 1996-3-13
-# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details)
-# and by Perry Greenfield 2000-4-1 for numarray
-# and by Travis Oliphant  2005-8-22 for numpy
-
-import sys
-import numerictypes as _nt
-from umath import maximum, minimum, absolute, not_equal, isnan, isinf
-#from multiarray import format_longfloat, datetime_as_string, datetime_data
-from fromnumeric import ravel
-
-
-def product(x, y): return x*y
-
-_summaryEdgeItems = 3     # repr N leading and trailing items of each dimension
-_summaryThreshold = 1000 # total items > triggers array summarization
-
-_float_output_precision = 8
-_float_output_suppress_small = False
-_line_width = 75
-_nan_str = 'nan'
-_inf_str = 'inf'
-_formatter = None  # formatting function for array elements
-
-if sys.version_info[0] >= 3:
-    from functools import reduce
-
-def set_printoptions(precision=None, threshold=None, edgeitems=None,
-                     linewidth=None, suppress=None,
-                     nanstr=None, infstr=None,
-                     formatter=None):
-    """
-    Set printing options.
-
-    These options determine the way floating point numbers, arrays and
-    other NumPy objects are displayed.
-
-    Parameters
-    ----------
-    precision : int, optional
-        Number of digits of precision for floating point output (default 8).
-    threshold : int, optional
-        Total number of array elements which trigger summarization
-        rather than full repr (default 1000).
-    edgeitems : int, optional
-        Number of array items in summary at beginning and end of
-        each dimension (default 3).
-    linewidth : int, optional
-        The number of characters per line for the purpose of inserting
-        line breaks (default 75).
-    suppress : bool, optional
-        Whether or not suppress printing of small floating point values
-        using scientific notation (default False).
-    nanstr : str, optional
-        String representation of floating point not-a-number (default nan).
-    infstr : str, optional
-        String representation of floating point infinity (default inf).
-    formatter : dict of callables, optional
-        If not None, the keys should indicate the type(s) that the respective
-        formatting function applies to.  Callables should return a string.
-        Types that are not specified (by their corresponding keys) are handled
-        by the default formatters.  Individual types for which a formatter
-        can be set are::
-
-            - 'bool'
-            - 'int'
-            - 'timedelta' : a `numpy.timedelta64`
-            - 'datetime' : a `numpy.datetime64`
-            - 'float'
-            - 'longfloat' : 128-bit floats
-            - 'complexfloat'
-            - 'longcomplexfloat' : composed of two 128-bit floats
-            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
-            - 'str' : all other strings
-
-        Other keys that can be used to set a group of types at once are::
-
-            - 'all' : sets all types
-            - 'int_kind' : sets 'int'
-            - 'float_kind' : sets 'float' and 'longfloat'
-            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
-            - 'str_kind' : sets 'str' and 'numpystr'
-
-    See Also
-    --------
-    get_printoptions, set_string_function, array2string
-
-    Notes
-    -----
-    `formatter` is always reset with a call to `set_printoptions`.
-
-    Examples
-    --------
-    Floating point precision can be set:
-
-    >>> np.set_printoptions(precision=4)
-    >>> print np.array([1.123456789])
-    [ 1.1235]
-
-    Long arrays can be summarised:
-
-    >>> np.set_printoptions(threshold=5)
-    >>> print np.arange(10)
-    [0 1 2 ..., 7 8 9]
-
-    Small results can be suppressed:
-
-    >>> eps = np.finfo(float).eps
-    >>> x = np.arange(4.)
-    >>> x**2 - (x + eps)**2
-    array([ -4.9304e-32,  -4.4409e-16,   0.0000e+00,   0.0000e+00])
-    >>> np.set_printoptions(suppress=True)
-    >>> x**2 - (x + eps)**2
-    array([-0., -0.,  0.,  0.])
-
-    A custom formatter can be used to display array elements as desired:
-
-    >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)})
-    >>> x = np.arange(3)
-    >>> x
-    array([int: 0, int: -1, int: -2])
-    >>> np.set_printoptions()  # formatter gets reset
-    >>> x
-    array([0, 1, 2])
-
-    To put back the default options, you can use:
-
-    >>> np.set_printoptions(edgeitems=3,infstr='inf',
-    ... linewidth=75, nanstr='nan', precision=8,
-    ... suppress=False, threshold=1000, formatter=None)
-    """
-
-    global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \
-           _line_width, _float_output_suppress_small, _nan_str, _inf_str, \
-           _formatter
-    if linewidth is not None:
-        _line_width = linewidth
-    if threshold is not None:
-        _summaryThreshold = threshold
-    if edgeitems is not None:
-        _summaryEdgeItems = edgeitems
-    if precision is not None:
-        _float_output_precision = precision
-    if suppress is not None:
-        _float_output_suppress_small = not not suppress
-    if nanstr is not None:
-        _nan_str = nanstr
-    if infstr is not None:
-        _inf_str = infstr
-    _formatter = formatter
-
-def get_printoptions():
-    """
-    Return the current print options.
-
-    Returns
-    -------
-    print_opts : dict
-        Dictionary of current print options with keys
-
-          - precision : int
-          - threshold : int
-          - edgeitems : int
-          - linewidth : int
-          - suppress : bool
-          - nanstr : str
-          - infstr : str
-          - formatter : dict of callables
-
-        For a full description of these options, see `set_printoptions`.
-
-    See Also
-    --------
-    set_printoptions, set_string_function
-
-    """
-    d = dict(precision=_float_output_precision,
-             threshold=_summaryThreshold,
-             edgeitems=_summaryEdgeItems,
-             linewidth=_line_width,
-             suppress=_float_output_suppress_small,
-             nanstr=_nan_str,
-             infstr=_inf_str,
-             formatter=_formatter)
-    return d
-
-def _leading_trailing(a):
-    import numeric as _nc
-    if a.ndim == 1:
-        if len(a) > 2*_summaryEdgeItems:
-            b = _nc.concatenate((a[:_summaryEdgeItems],
-                                     a[-_summaryEdgeItems:]))
-        else:
-            b = a
-    else:
-        if len(a) > 2*_summaryEdgeItems:
-            l = [_leading_trailing(a[i]) for i in range(
-                min(len(a), _summaryEdgeItems))]
-            l.extend([_leading_trailing(a[-i]) for i in range(
-                min(len(a), _summaryEdgeItems),0,-1)])
-        else:
-            l = [_leading_trailing(a[i]) for i in range(0, len(a))]
-        b = _nc.concatenate(tuple(l))
-    return b
-
-def _boolFormatter(x):
-    if x:
-        return ' True'
-    else:
-        return 'False'
-
-
-def repr_format(x):
-    return repr(x)
-
-def _array2string(a, max_line_width, precision, suppress_small, separator=' ',
-                  prefix="", formatter=None):
-
-    if max_line_width is None:
-        max_line_width = _line_width
-
-    if precision is None:
-        precision = _float_output_precision
-
-    if suppress_small is None:
-        suppress_small = _float_output_suppress_small
-
-    if formatter is None:
-        formatter = _formatter
-
-    if a.size > _summaryThreshold:
-        summary_insert = "..., "
-        data = _leading_trailing(a)
-    else:
-        summary_insert = ""
-        data = ravel(a)
-
-    formatdict = {'bool' : _boolFormatter,
-                  'int' : IntegerFormat(data),
-                  'float' : FloatFormat(data, precision, suppress_small),
-                  'longfloat' : FloatFormat(data, precision, suppress_small),
-                  'complexfloat' : ComplexFormat(data, precision,
-                                                 suppress_small),
-                  'longcomplexfloat' : ComplexFormat(data, precision,
-                                                     suppress_small),
-                  'datetime' : DatetimeFormat(data),
-                  'timedelta' : TimedeltaFormat(data),
-                  'numpystr' : repr_format,
-                  'str' : str}
-
-    if formatter is not None:
-        fkeys = [k for k in formatter.keys() if formatter[k] is not None]
-        if 'all' in fkeys:
-            for key in formatdict.keys():
-                formatdict[key] = formatter['all']
-        if 'int_kind' in fkeys:
-            for key in ['int']:
-                formatdict[key] = formatter['int_kind']
-        if 'float_kind' in fkeys:
-            for key in ['float', 'longfloat']:
-                formatdict[key] = formatter['float_kind']
-        if 'complex_kind' in fkeys:
-            for key in ['complexfloat', 'longcomplexfloat']:
-                formatdict[key] = formatter['complex_kind']
-        if 'str_kind' in fkeys:
-            for key in ['numpystr', 'str']:
-                formatdict[key] = formatter['str_kind']
-        for key in formatdict.keys():
-            if key in fkeys:
-                formatdict[key] = formatter[key]
-
-    try:
-        format_function = a._format
-        msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \
-              "will be removed in 2.1. Use the `formatter` kw instead."
-        import warnings
-        warnings.warn(msg, DeprecationWarning)
-    except AttributeError:
-        # find the right formatting function for the array
-        dtypeobj = a.dtype.type
-        if issubclass(dtypeobj, _nt.bool_):
-            format_function = formatdict['bool']
-        elif issubclass(dtypeobj, _nt.integer):
-            #if issubclass(dtypeobj, _nt.timedelta64):
-            #    format_function = formatdict['timedelta']
-            #else:
-            format_function = formatdict['int']
-        elif issubclass(dtypeobj, _nt.floating):
-            if hasattr(_nt, 'longfloat') and issubclass(dtypeobj, _nt.longfloat):
-                format_function = formatdict['longfloat']
-            else:
-                format_function = formatdict['float']
-        elif issubclass(dtypeobj, _nt.complexfloating):
-            if hasattr(_nt, 'clongfloat') and issubclass(dtypeobj, _nt.clongfloat):
-                format_function = formatdict['longcomplexfloat']
-            else:
-                format_function = formatdict['complexfloat']
-        elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)):
-            format_function = formatdict['numpystr']
-        #elif issubclass(dtypeobj, _nt.datetime64):
-        #    format_function = formatdict['datetime']
-        else:
-            format_function = formatdict['str']
-
-    # skip over "["
-    next_line_prefix = " "
-    # skip over array(
-    next_line_prefix += " "*len(prefix)
-
-    lst = _formatArray(a, format_function, len(a.shape), max_line_width,
-                       next_line_prefix, separator,
-                       _summaryEdgeItems, summary_insert)[:-1]
-    return lst
-
-def _convert_arrays(obj):
-    import numeric as _nc
-    newtup = []
-    for k in obj:
-        if isinstance(k, _nc.ndarray):
-            k = k.tolist()
-        elif isinstance(k, tuple):
-            k = _convert_arrays(k)
-        newtup.append(k)
-    return tuple(newtup)
-
-
-def array2string(a, max_line_width=None, precision=None,
-                 suppress_small=None, separator=' ', prefix="",
-                 style=repr, formatter=None):
-    """
-    Return a string representation of an array.
-
-    Parameters
-    ----------
-    a : ndarray
-        Input array.
-    max_line_width : int, optional
-        The maximum number of columns the string should span. Newline
-        characters splits the string appropriately after array elements.
-    precision : int, optional
-        Floating point precision. Default is the current printing
-        precision (usually 8), which can be altered using `set_printoptions`.
-    suppress_small : bool, optional
-        Represent very small numbers as zero. A number is "very small" if it
-        is smaller than the current printing precision.
-    separator : str, optional
-        Inserted between elements.
-    prefix : str, optional
-        An array is typically printed as::
-
-          'prefix(' + array2string(a) + ')'
-
-        The length of the prefix string is used to align the
-        output correctly.
-    style : function, optional
-        A function that accepts an ndarray and returns a string.  Used only
-        when the shape of `a` is equal to ``()``, i.e. for 0-D arrays.
-    formatter : dict of callables, optional
-        If not None, the keys should indicate the type(s) that the respective
-        formatting function applies to.  Callables should return a string.
-        Types that are not specified (by their corresponding keys) are handled
-        by the default formatters.  Individual types for which a formatter
-        can be set are::
-
-            - 'bool'
-            - 'int'
-            - 'timedelta' : a `numpy.timedelta64`
-            - 'datetime' : a `numpy.datetime64`
-            - 'float'
-            - 'longfloat' : 128-bit floats
-            - 'complexfloat'
-            - 'longcomplexfloat' : composed of two 128-bit floats
-            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
-            - 'str' : all other strings
-
-        Other keys that can be used to set a group of types at once are::
-
-            - 'all' : sets all types
-            - 'int_kind' : sets 'int'
-            - 'float_kind' : sets 'float' and 'longfloat'
-            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
-            - 'str_kind' : sets 'str' and 'numpystr'
-
-    Returns
-    -------
-    array_str : str
-        String representation of the array.
-
-    Raises
-    ------
-    TypeError : if a callable in `formatter` does not return a string.
-
-    See Also
-    --------
-    array_str, array_repr, set_printoptions, get_printoptions
-
-    Notes
-    -----
-    If a formatter is specified for a certain type, the `precision` keyword is
-    ignored for that type.
-
-    Examples
-    --------
-    >>> x = np.array([1e-16,1,2,3])
-    >>> print np.array2string(x, precision=2, separator=',',
-    ...                       suppress_small=True)
-    [ 0., 1., 2., 3.]
-
-    >>> x  = np.arange(3.)
-    >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x})
-    '[0.00 1.00 2.00]'
-
-    >>> x  = np.arange(3)
-    >>> np.array2string(x, formatter={'int':lambda x: hex(x)})
-    '[0x0L 0x1L 0x2L]'
-
-    """
-
-    if a.shape == ():
-        x = a.item()
-        try:
-            lst = a._format(x)
-            msg = "The `_format` attribute is deprecated in Numpy " \
-                  "2.0 and will be removed in 2.1. Use the " \
-                  "`formatter` kw instead."
-            import warnings
-            warnings.warn(msg, DeprecationWarning)
-        except AttributeError:
-            if isinstance(x, tuple):
-                x = _convert_arrays(x)
-            lst = style(x)
-    elif reduce(product, a.shape) == 0:
-        # treat as a null array if any of shape elements == 0
-        lst = "[]"
-    else:
-        lst = _array2string(a, max_line_width, precision, suppress_small,
-                            separator, prefix, formatter=formatter)
-    return lst
-
-def _extendLine(s, line, word, max_line_len, next_line_prefix):
-    if len(line.rstrip()) + len(word.rstrip()) >= max_line_len:
-        s += line.rstrip() + "\n"
-        line = next_line_prefix
-    line += word
-    return s, line
-
-
-def _formatArray(a, format_function, rank, max_line_len,
-                 next_line_prefix, separator, edge_items, summary_insert):
-    """formatArray is designed for two modes of operation:
-
-    1. Full output
-
-    2. Summarized output
-
-    """
-    if rank == 0:
-        obj = a.item()
-        if isinstance(obj, tuple):
-            obj = _convert_arrays(obj)
-        return str(obj)
-
-    if summary_insert and 2*edge_items < len(a):
-        leading_items, trailing_items, summary_insert1 = \
-                       edge_items, edge_items, summary_insert
-    else:
-        leading_items, trailing_items, summary_insert1 = 0, len(a), ""
-
-    if rank == 1:
-        s = ""
-        line = next_line_prefix
-        for i in xrange(leading_items):
-            word = format_function(a[i]) + separator
-            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-
-        if summary_insert1:
-            s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix)
-
-        for i in xrange(trailing_items, 1, -1):
-            word = format_function(a[-i]) + separator
-            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-
-        word = format_function(a[-1])
-        s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-        s += line + "]\n"
-        s = '[' + s[len(next_line_prefix):]
-    else:
-        s = '['
-        sep = separator.rstrip()
-        for i in xrange(leading_items):
-            if i > 0:
-                s += next_line_prefix
-            s += _formatArray(a[i], format_function, rank-1, max_line_len,
-                              " " + next_line_prefix, separator, edge_items,
-                              summary_insert)
-            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
-
-        if summary_insert1:
-            s += next_line_prefix + summary_insert1 + "\n"
-
-        for i in xrange(trailing_items, 1, -1):
-            if leading_items or i != trailing_items:
-                s += next_line_prefix
-            s += _formatArray(a[-i], format_function, rank-1, max_line_len,
-                              " " + next_line_prefix, separator, edge_items,
-                              summary_insert)
-            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
-        if leading_items or trailing_items > 1:
-            s += next_line_prefix
-        s += _formatArray(a[-1], format_function, rank-1, max_line_len,
-                          " " + next_line_prefix, separator, edge_items,
-                          summary_insert).rstrip()+']\n'
-    return s
-
-class FloatFormat(object):
-    def __init__(self, data, precision, suppress_small, sign=False):
-        self.precision = precision
-        self.suppress_small = suppress_small
-        self.sign = sign
-        self.exp_format = False
-        self.large_exponent = False
-        self.max_str_len = 0
-        try:
-            self.fillFormat(data)
-        except (TypeError, NotImplementedError):
-            # if reduce(data) fails, this instance will not be called, just
-            # instantiated in formatdict.
-            pass
-
-    def fillFormat(self, data):
-        import numeric as _nc
-        errstate = _nc.seterr(all='ignore')
-        try:
-            special = isnan(data) | isinf(data)
-            valid = not_equal(data, 0) & ~special
-            non_zero = absolute(data.compress(valid))
-            if len(non_zero) == 0:
-                max_val = 0.
-                min_val = 0.
-            else:
-                max_val = maximum.reduce(non_zero)
-                min_val = minimum.reduce(non_zero)
-                if max_val >= 1.e8:
-                    self.exp_format = True
-                if not self.suppress_small and (min_val < 0.0001
-                                           or max_val/min_val > 1000.):
-                    self.exp_format = True
-        finally:
-            _nc.seterr(**errstate)
-
-        if self.exp_format:
-            self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100
-            self.max_str_len = 8 + self.precision
-            if self.large_exponent:
-                self.max_str_len += 1
-            if self.sign:
-                format = '%+'
-            else:
-                format = '%'
-            format = format + '%d.%de' % (self.max_str_len, self.precision)
-        else:
-            format = '%%.%df' % (self.precision,)
-            if len(non_zero):
-                precision = max([_digits(x, self.precision, format)
-                                 for x in non_zero])
-            else:
-                precision = 0
-            precision = min(self.precision, precision)
-            self.max_str_len = len(str(int(max_val))) + precision + 2
-            if _nc.any(special):
-                self.max_str_len = max(self.max_str_len,
-                                       len(_nan_str),
-                                       len(_inf_str)+1)
-            if self.sign:
-                format = '%#+'
-            else:
-                format = '%#'
-            format = format + '%d.%df' % (self.max_str_len, precision)
-
-        self.special_fmt = '%%%ds' % (self.max_str_len,)
-        self.format = format
-
-    def __call__(self, x, strip_zeros=True):
-        import numeric as _nc
-        err = _nc.seterr(invalid='ignore')
-        try:
-            if isnan(x):
-                if self.sign:
-                    return self.special_fmt % ('+' + _nan_str,)
-                else:
-                    return self.special_fmt % (_nan_str,)
-            elif isinf(x):
-                if x > 0:
-                    if self.sign:
-                        return self.special_fmt % ('+' + _inf_str,)
-                    else:
-                        return self.special_fmt % (_inf_str,)
-                else:
-                    return self.special_fmt % ('-' + _inf_str,)
-        finally:
-            _nc.seterr(**err)
-
-        s = self.format % x
-        if self.large_exponent:
-            # 3-digit exponent
-            expsign = s[-3]
-            if expsign == '+' or expsign == '-':
-                s = s[1:-2] + '0' + s[-2:]
-        elif self.exp_format:
-            # 2-digit exponent
-            if s[-3] == '0':
-                s = ' ' + s[:-3] + s[-2:]
-        elif strip_zeros:
-            z = s.rstrip('0')
-            s = z + ' '*(len(s)-len(z))
-        return s
-
-
-def _digits(x, precision, format):
-    s = format % x
-    z = s.rstrip('0')
-    return precision - len(s) + len(z)
-
-
-_MAXINT = sys.maxint
-_MININT = -sys.maxint-1
-class IntegerFormat(object):
-    def __init__(self, data):
-        try:
-            max_str_len = max(len(str(maximum.reduce(data))),
-                              len(str(minimum.reduce(data))))
-            self.format = '%' + str(max_str_len) + 'd'
-        except (TypeError, NotImplementedError):
-            # if reduce(data) fails, this instance will not be called, just
-            # instantiated in formatdict.
-            pass
-        except ValueError:
-            # this occurs when everything is NA
-            pass
-
-    def __call__(self, x):
-        if _MININT < x < _MAXINT:
-            return self.format % x
-        else:
-            return "%s" % x
-
-class LongFloatFormat(object):
-    # XXX Have to add something to determine the width to use a la FloatFormat
-    # Right now, things won't line up properly
-    def __init__(self, precision, sign=False):
-        self.precision = precision
-        self.sign = sign
-
-    def __call__(self, x):
-        if isnan(x):
-            if self.sign:
-                return '+' + _nan_str
-            else:
-                return ' ' + _nan_str
-        elif isinf(x):
-            if x > 0:
-                if self.sign:
-                    return '+' + _inf_str
-                else:
-                    return ' ' + _inf_str
-            else:
-                return '-' + _inf_str
-        elif x >= 0:
-            if self.sign:
-                return '+' + format_longfloat(x, self.precision)
-            else:
-                return ' ' + format_longfloat(x, self.precision)
-        else:
-            return format_longfloat(x, self.precision)
-
-
-class LongComplexFormat(object):
-    def __init__(self, precision):
-        self.real_format = LongFloatFormat(precision)
-        self.imag_format = LongFloatFormat(precision, sign=True)
-
-    def __call__(self, x):
-        r = self.real_format(x.real)
-        i = self.imag_format(x.imag)
-        return r + i + 'j'
-
-
-class ComplexFormat(object):
-    def __init__(self, x, precision, suppress_small):
-        self.real_format = FloatFormat(x.real, precision, suppress_small)
-        self.imag_format = FloatFormat(x.imag, precision, suppress_small,
-                                       sign=True)
-
-    def __call__(self, x):
-        r = self.real_format(x.real, strip_zeros=False)
-        i = self.imag_format(x.imag, strip_zeros=False)
-        if not self.imag_format.exp_format:
-            z = i.rstrip('0')
-            i = z + 'j' + ' '*(len(i)-len(z))
-        else:
-            i = i + 'j'
-        return r + i
-
-class DatetimeFormat(object):
-    def __init__(self, x, unit=None,
-                timezone=None, casting='same_kind'):
-        # Get the unit from the dtype
-        if unit is None:
-            if x.dtype.kind == 'M':
-                unit = datetime_data(x.dtype)[0]
-            else:
-                unit = 's'
-
-        # If timezone is default, make it 'local' or 'UTC' based on the unit
-        if timezone is None:
-            # Date units -> UTC, time units -> local
-            if unit in ('Y', 'M', 'W', 'D'):
-                self.timezone = 'UTC'
-            else:
-                self.timezone = 'local'
-        else:
-            self.timezone = timezone
-        self.unit = unit
-        self.casting = casting
-
-    def __call__(self, x):
-        return "'%s'" % datetime_as_string(x,
-                                    unit=self.unit,
-                                    timezone=self.timezone,
-                                    casting=self.casting)
-
-class TimedeltaFormat(object):
-    def __init__(self, data):
-        if data.dtype.kind == 'm':
-            v = data.view('i8')
-            max_str_len = max(len(str(maximum.reduce(v))),
-                              len(str(minimum.reduce(v))))
-            self.format = '%' + str(max_str_len) + 'd'
-
-    def __call__(self, x):
-        return self.format % x.astype('i8')
-
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ /dev/null
@@ -1,2924 +0,0 @@
-######################################################################
-# This is a copy of numpy/core/fromnumeric.py modified for numpypy
-######################################################################
-"""Module containing non-deprecated functions borrowed from Numeric.
-
-"""
-from __future__ import division, absolute_import, print_function
-
-import types
-
-from . import multiarray as mu
-from . import umath as um
-from . import numerictypes as nt
-from .numeric import asarray, array, asanyarray, concatenate
-from . import _methods
-
-
-# functions that are methods
-__all__ = [
-        'alen', 'all', 'alltrue', 'amax', 'amin', 'any', 'argmax',
-        'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip',
-        'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean',
-        'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put',
-        'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_',
-        'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze',
-        'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var',
-        ]
-
-
-try:
-    _gentype = types.GeneratorType
-except AttributeError:
-    _gentype = type(None)
-
-# save away Python sum
-_sum_ = sum
-
-# functions that are now methods
-def _wrapit(obj, method, *args, **kwds):
-    try:
-        wrap = obj.__array_wrap__
-    except AttributeError:
-        wrap = None
-    result = getattr(asarray(obj), method)(*args, **kwds)
-    if wrap:
-        if not isinstance(result, mu.ndarray):
-            result = asarray(result)
-        result = wrap(result)
-    return result
-
-
-def take(a, indices, axis=None, out=None, mode='raise'):
-    """
-    Take elements from an array along an axis.
-
-    This function does the same thing as "fancy" indexing (indexing arrays
-    using arrays); however, it can be easier to use if you need elements
-    along a given axis.
-
-    Parameters
-    ----------
-    a : array_like
-        The source array.
-    indices : array_like
-        The indices of the values to extract.
-
-        .. versionadded:: 1.8.0
-
-        Also allow scalars for indices.
-    axis : int, optional
-        The axis over which to select values. By default, the flattened
-        input array is used.
-    out : ndarray, optional
-        If provided, the result will be placed in this array. It should
-        be of the appropriate shape and dtype.
-    mode : {'raise', 'wrap', 'clip'}, optional
-        Specifies how out-of-bounds indices will behave.
-
-        * 'raise' -- raise an error (default)
-        * 'wrap' -- wrap around
-        * 'clip' -- clip to the range
-
-        'clip' mode means that all indices that are too large are replaced
-        by the index that addresses the last element along that axis. Note
-        that this disables indexing with negative numbers.
-
-    Returns
-    -------
-    subarray : ndarray
-        The returned array has the same type as `a`.
-
-    See Also
-    --------
-    ndarray.take : equivalent method
-
-    Examples
-    --------
-    >>> a = [4, 3, 5, 7, 6, 8]
-    >>> indices = [0, 1, 4]
-    >>> np.take(a, indices)
-    array([4, 3, 6])
-
-    In this example if `a` is an ndarray, "fancy" indexing can be used.
-
-    >>> a = np.array(a)
-    >>> a[indices]
-    array([4, 3, 6])
-
-    If `indices` is not one dimensional, the output also has these dimensions.
-
-    >>> np.take(a, [[0, 1], [2, 3]])
-    array([[4, 3],
-           [5, 7]])
-    """
-    try:
-        take = a.take
-    except AttributeError:
-        return _wrapit(a, 'take', indices, axis, out, mode)
-    return take(indices, axis, out, mode)
-
-
-# not deprecated --- copy if necessary, view otherwise
-def reshape(a, newshape, order='C'):
-    """
-    Gives a new shape to an array without changing its data.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be reshaped.
-    newshape : int or tuple of ints
-        The new shape should be compatible with the original shape. If
-        an integer, then the result will be a 1-D array of that length.
-        One shape dimension can be -1. In this case, the value is inferred
-        from the length of the array and remaining dimensions.
-    order : {'C', 'F', 'A'}, optional
-        Read the elements of `a` using this index order, and place the elements
-        into the reshaped array using this index order.  'C' means to
-        read / write the elements using C-like index order, with the last axis index
-        changing fastest, back to the first axis index changing slowest.  'F'
-        means to read / write the elements using Fortran-like index order, with
-        the first index changing fastest, and the last index changing slowest.
-        Note that the 'C' and 'F' options take no account of the memory layout
-        of the underlying array, and only refer to the order of indexing.  'A'
-        means to read / write the elements in Fortran-like index order if `a` is
-        Fortran *contiguous* in memory, C-like order otherwise.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        This will be a new view object if possible; otherwise, it will
-        be a copy.  Note there is no guarantee of the *memory layout* (C- or
-        Fortran- contiguous) of the returned array.
-
-    See Also
-    --------
-    ndarray.reshape : Equivalent method.
-
-    Notes
-    -----
-    It is not always possible to change the shape of an array without
-    copying the data. If you want an error to be raise if the data is copied,
-    you should assign the new shape to the shape attribute of the array::
-
-     >>> a = np.zeros((10, 2))
-     # A transpose make the array non-contiguous
-     >>> b = a.T
-     # Taking a view makes it possible to modify the shape without modifying the
-     # initial object.
-     >>> c = b.view()
-     >>> c.shape = (20)
-     AttributeError: incompatible shape for a non-contiguous array
-
-    The `order` keyword gives the index ordering both for *fetching* the values
-    from `a`, and then *placing* the values into the output array.  For example,
-    let's say you have an array:
-
-    >>> a = np.arange(6).reshape((3, 2))
-    >>> a
-    array([[0, 1],
-           [2, 3],
-           [4, 5]])
-
-    You can think of reshaping as first raveling the array (using the given
-    index order), then inserting the elements from the raveled array into the
-    new array using the same kind of index ordering as was used for the
-    raveling.
-
-    >>> np.reshape(a, (2, 3)) # C-like index ordering
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.reshape(a, (2, 3), order='F') # Fortran-like index ordering
-    array([[0, 4, 3],
-           [2, 1, 5]])
-    >>> np.reshape(np.ravel(a, order='F'), (2, 3), order='F')
-    array([[0, 4, 3],
-           [2, 1, 5]])
-
-    Examples
-    --------
-    >>> a = np.array([[1,2,3], [4,5,6]])
-    >>> np.reshape(a, 6)
-    array([1, 2, 3, 4, 5, 6])
-    >>> np.reshape(a, 6, order='F')
-    array([1, 4, 2, 5, 3, 6])
-
-    >>> np.reshape(a, (3,-1))       # the unspecified value is inferred to be 2
-    array([[1, 2],
-           [3, 4],
-           [5, 6]])
-    """
-    assert order == 'C'
-    try:
-        reshape = a.reshape
-    except AttributeError:
-        return _wrapit(a, 'reshape', newshape)
-    return reshape(newshape)
-
-
-def choose(a, choices, out=None, mode='raise'):
-    """
-    Construct an array from an index array and a set of arrays to choose from.
-
-    First of all, if confused or uncertain, definitely look at the Examples -
-    in its full generality, this function is less simple than it might
-    seem from the following code description (below ndi =
-    `numpy.lib.index_tricks`):
-
-    ``np.choose(a,c) == np.array([c[a[I]][I] for I in ndi.ndindex(a.shape)])``.
-
-    But this omits some subtleties.  Here is a fully general summary:
-
-    Given an "index" array (`a`) of integers and a sequence of `n` arrays
-    (`choices`), `a` and each choice array are first broadcast, as necessary,
-    to arrays of a common shape; calling these *Ba* and *Bchoices[i], i =
-    0,...,n-1* we have that, necessarily, ``Ba.shape == Bchoices[i].shape``
-    for each `i`.  Then, a new array with shape ``Ba.shape`` is created as
-    follows:
-
-    * if ``mode=raise`` (the default), then, first of all, each element of
-      `a` (and thus `Ba`) must be in the range `[0, n-1]`; now, suppose that
-      `i` (in that range) is the value at the `(j0, j1, ..., jm)` position
-      in `Ba` - then the value at the same position in the new array is the
-      value in `Bchoices[i]` at that same position;
-
-    * if ``mode=wrap``, values in `a` (and thus `Ba`) may be any (signed)
-      integer; modular arithmetic is used to map integers outside the range
-      `[0, n-1]` back into that range; and then the new array is constructed
-      as above;
-
-    * if ``mode=clip``, values in `a` (and thus `Ba`) may be any (signed)
-      integer; negative integers are mapped to 0; values greater than `n-1`
-      are mapped to `n-1`; and then the new array is constructed as above.
-
-    Parameters
-    ----------
-    a : int array
-        This array must contain integers in `[0, n-1]`, where `n` is the number
-        of choices, unless ``mode=wrap`` or ``mode=clip``, in which cases any
-        integers are permissible.
-    choices : sequence of arrays
-        Choice arrays. `a` and all of the choices must be broadcastable to the
-        same shape.  If `choices` is itself an array (not recommended), then
-        its outermost dimension (i.e., the one corresponding to
-        ``choices.shape[0]``) is taken as defining the "sequence".
-    out : array, optional
-        If provided, the result will be inserted into this array. It should
-        be of the appropriate shape and dtype.
-    mode : {'raise' (default), 'wrap', 'clip'}, optional
-        Specifies how indices outside `[0, n-1]` will be treated:
-
-          * 'raise' : an exception is raised
-          * 'wrap' : value becomes value mod `n`
-          * 'clip' : values < 0 are mapped to 0, values > n-1 are mapped to n-1
-
-    Returns
-    -------
-    merged_array : array
-        The merged result.
-
-    Raises
-    ------
-    ValueError: shape mismatch
-        If `a` and each choice array are not all broadcastable to the same
-        shape.
-
-    See Also
-    --------
-    ndarray.choose : equivalent method
-
-    Notes
-    -----
-    To reduce the chance of misinterpretation, even though the following
-    "abuse" is nominally supported, `choices` should neither be, nor be
-    thought of as, a single array, i.e., the outermost sequence-like container
-    should be either a list or a tuple.
-
-    Examples
-    --------
-
-    >>> choices = [[0, 1, 2, 3], [10, 11, 12, 13],
-    ...   [20, 21, 22, 23], [30, 31, 32, 33]]
-    >>> np.choose([2, 3, 1, 0], choices
-    ... # the first element of the result will be the first element of the
-    ... # third (2+1) "array" in choices, namely, 20; the second element
-    ... # will be the second element of the fourth (3+1) choice array, i.e.,
-    ... # 31, etc.
-    ... )
-    array([20, 31, 12,  3])
-    >>> np.choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1)
-    array([20, 31, 12,  3])
-    >>> # because there are 4 choice arrays
-    >>> np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4)
-    array([20,  1, 12,  3])
-    >>> # i.e., 0
-
-    A couple examples illustrating how choose broadcasts:
-
-    >>> a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]]
-    >>> choices = [-10, 10]
-    >>> np.choose(a, choices)
-    array([[ 10, -10,  10],
-           [-10,  10, -10],
-           [ 10, -10,  10]])
-
-    >>> # With thanks to Anne Archibald
-    >>> a = np.array([0, 1]).reshape((2,1,1))
-    >>> c1 = np.array([1, 2, 3]).reshape((1,3,1))
-    >>> c2 = np.array([-1, -2, -3, -4, -5]).reshape((1,1,5))
-    >>> np.choose(a, (c1, c2)) # result is 2x3x5, res[0,:,:]=c1, res[1,:,:]=c2
-    array([[[ 1,  1,  1,  1,  1],
-            [ 2,  2,  2,  2,  2],
-            [ 3,  3,  3,  3,  3]],
-           [[-1, -2, -3, -4, -5],
-            [-1, -2, -3, -4, -5],
-            [-1, -2, -3, -4, -5]]])
-
-    """
-    try:
-        choose = a.choose
-    except AttributeError:
-        return _wrapit(a, 'choose', choices, out=out, mode=mode)
-    return choose(choices, out=out, mode=mode)
-
-
-def repeat(a, repeats, axis=None):
-    """
-    Repeat elements of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    repeats : {int, array of ints}
-        The number of repetitions for each element.  `repeats` is broadcasted
-        to fit the shape of the given axis.
-    axis : int, optional
-        The axis along which to repeat values.  By default, use the
-        flattened input array, and return a flat output array.
-
-    Returns
-    -------
-    repeated_array : ndarray
-        Output array which has the same shape as `a`, except along
-        the given axis.
-
-    See Also
-    --------
-    tile : Tile an array.
-
-    Examples
-    --------
-    >>> x = np.array([[1,2],[3,4]])
-    >>> np.repeat(x, 2)
-    array([1, 1, 2, 2, 3, 3, 4, 4])
-    >>> np.repeat(x, 3, axis=1)
-    array([[1, 1, 1, 2, 2, 2],
-           [3, 3, 3, 4, 4, 4]])
-    >>> np.repeat(x, [1, 2], axis=0)
-    array([[1, 2],
-           [3, 4],
-           [3, 4]])
-
-    """
-    try:
-        repeat = a.repeat
-    except AttributeError:
-        return _wrapit(a, 'repeat', repeats, axis)
-    return repeat(repeats, axis)
-
-
-def put(a, ind, v, mode='raise'):
-    """
-    Replaces specified elements of an array with given values.
-
-    The indexing works on the flattened target array. `put` is roughly
-    equivalent to:
-
-    ::
-
-        a.flat[ind] = v
-
-    Parameters
-    ----------
-    a : ndarray
-        Target array.
-    ind : array_like
-        Target indices, interpreted as integers.
-    v : array_like
-        Values to place in `a` at target indices. If `v` is shorter than
-        `ind` it will be repeated as necessary.
-    mode : {'raise', 'wrap', 'clip'}, optional
-        Specifies how out-of-bounds indices will behave.
-
-        * 'raise' -- raise an error (default)
-        * 'wrap' -- wrap around
-        * 'clip' -- clip to the range
-
-        'clip' mode means that all indices that are too large are replaced
-        by the index that addresses the last element along that axis. Note
-        that this disables indexing with negative numbers.
-
-    See Also
-    --------
-    putmask, place
-
-    Examples
-    --------
-    >>> a = np.arange(5)
-    >>> np.put(a, [0, 2], [-44, -55])
-    >>> a
-    array([-44,   1, -55,   3,   4])
-
-    >>> a = np.arange(5)
-    >>> np.put(a, 22, -5, mode='clip')
-    >>> a
-    array([ 0,  1,  2,  3, -5])
-
-    """
-    return a.put(ind, v, mode)
-
-
-def swapaxes(a, axis1, axis2):
-    """
-    Interchange two axes of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axis1 : int
-        First axis.
-    axis2 : int
-        Second axis.
-
-    Returns
-    -------
-    a_swapped : ndarray
-        If `a` is an ndarray, then a view of `a` is returned; otherwise
-        a new array is created.
-
-    Examples
-    --------
-    >>> x = np.array([[1,2,3]])
-    >>> np.swapaxes(x,0,1)
-    array([[1],
-           [2],
-           [3]])
-
-    >>> x = np.array([[[0,1],[2,3]],[[4,5],[6,7]]])
-    >>> x
-    array([[[0, 1],
-            [2, 3]],
-           [[4, 5],
-            [6, 7]]])
-
-    >>> np.swapaxes(x,0,2)
-    array([[[0, 4],
-            [2, 6]],
-           [[1, 5],
-            [3, 7]]])
-
-    """
-    try:
-        swapaxes = a.swapaxes
-    except AttributeError:
-        return _wrapit(a, 'swapaxes', axis1, axis2)
-    return swapaxes(axis1, axis2)
-
-
-def transpose(a, axes=None):
-    """
-    Permute the dimensions of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axes : list of ints, optional
-        By default, reverse the dimensions, otherwise permute the axes
-        according to the values given.
-
-    Returns
-    -------
-    p : ndarray
-        `a` with its axes permuted.  A view is returned whenever
-        possible.
-
-    See Also
-    --------
-    rollaxis
-
-    Examples
-    --------
-    >>> x = np.arange(4).reshape((2,2))
-    >>> x
-    array([[0, 1],
-           [2, 3]])
-
-    >>> np.transpose(x)
-    array([[0, 2],
-           [1, 3]])
-
-    >>> x = np.ones((1, 2, 3))
-    >>> np.transpose(x, (1, 0, 2)).shape
-    (2, 1, 3)
-
-    """
-    if axes is not None:
-        raise NotImplementedError('No "axes" arg yet.')
-    try:
-        transpose = a.transpose
-    except AttributeError:
-        return _wrapit(a, 'transpose')
-    return transpose()
-
-
-def partition(a, kth, axis=-1, kind='introselect', order=None):
-    """
-    Return a partitioned copy of an array.
-
-    Creates a copy of the array with its elements rearranged in such a way that
-    the value of the element in kth position is in the position it would be in
-    a sorted array. All elements smaller than the kth element are moved before
-    this element and all equal or greater are moved behind it. The ordering of
-    the elements in the two partitions is undefined.
-
-    .. versionadded:: 1.8.0
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be sorted.
-    kth : int or sequence of ints
-        Element index to partition by. The kth value of the element will be in
-        its final sorted position and all smaller elements will be moved before
-        it and all equal or greater elements behind it.
-        The order all elements in the partitions is undefined.
-        If provided with a sequence of kth it will partition all elements
-        indexed by kth  of them into their sorted position at once.
-    axis : int or None, optional
-        Axis along which to sort. If None, the array is flattened before
-        sorting. The default is -1, which sorts along the last axis.
-    kind : {'introselect'}, optional
-        Selection algorithm. Default is 'introselect'.
-    order : list, optional
-        When `a` is a structured array, this argument specifies which fields
-        to compare first, second, and so on.  This list does not need to
-        include all of the fields.
-
-    Returns
-    -------
-    partitioned_array : ndarray
-        Array of the same type and shape as `a`.
-
-    See Also
-    --------
-    ndarray.partition : Method to sort an array in-place.
-    argpartition : Indirect partition.
-    sort : Full sorting
-
-    Notes
-    -----
-    The various selection algorithms are characterized by their average speed,
-    worst case performance, work space size, and whether they are stable. A
-    stable sort keeps items with the same key in the same relative order. The
-    three available algorithms have the following properties:
-
-    ================= ======= ============= ============ =======
-       kind            speed   worst case    work space  stable
-    ================= ======= ============= ============ =======
-    'introselect'        1        O(n)           0         no
-    ================= ======= ============= ============ =======
-
-    All the partition algorithms make temporary copies of the data when
-    partitioning along any but the last axis.  Consequently, partitioning
-    along the last axis is faster and uses less space than partitioning
-    along any other axis.
-
-    The sort order for complex numbers is lexicographic. If both the real
-    and imaginary parts are non-nan then the order is determined by the
-    real parts except when they are equal, in which case the order is
-    determined by the imaginary parts.
-
-    Examples
-    --------
-    >>> a = np.array([3, 4, 2, 1])
-    >>> np.partition(a, 3)
-    array([2, 1, 3, 4])
-
-    >>> np.partition(a, (1, 3))
-    array([1, 2, 3, 4])
-
-    """
-    if axis is None:
-        a = asanyarray(a).flatten()
-        axis = 0
-    else:
-        a = asanyarray(a).copy()
-    a.partition(kth, axis=axis, kind=kind, order=order)
-    return a
-
-
-def argpartition(a, kth, axis=-1, kind='introselect', order=None):
-    """
-    Perform an indirect partition along the given axis using the algorithm
-    specified by the `kind` keyword. It returns an array of indices of the
-    same shape as `a` that index data along the given axis in partitioned
-    order.
-
-    .. versionadded:: 1.8.0
-
-    Parameters
-    ----------
-    a : array_like
-        Array to sort.
-    kth : int or sequence of ints
-        Element index to partition by. The kth element will be in its final
-        sorted position and all smaller elements will be moved before it and
-        all larger elements behind it.
-        The order all elements in the partitions is undefined.
-        If provided with a sequence of kth it will partition all of them into
-        their sorted position at once.
-    axis : int or None, optional
-        Axis along which to sort.  The default is -1 (the last axis). If None,
-        the flattened array is used.
-    kind : {'introselect'}, optional
-        Selection algorithm. Default is 'introselect'
-    order : list, optional
-        When `a` is an array with fields defined, this argument specifies
-        which fields to compare first, second, etc.  Not all fields need be
-        specified.
-
-    Returns
-    -------
-    index_array : ndarray, int
-        Array of indices that partition `a` along the specified axis.
-        In other words, ``a[index_array]`` yields a sorted `a`.
-
-    See Also
-    --------
-    partition : Describes partition algorithms used.
-    ndarray.partition : Inplace partition.
-    argsort : Full indirect sort
-
-    Notes
-    -----
-    See `partition` for notes on the different selection algorithms.
-
-    Examples
-    --------
-    One dimensional array:
-
-    >>> x = np.array([3, 4, 2, 1])
-    >>> x[np.argpartition(x, 3)]
-    array([2, 1, 3, 4])
-    >>> x[np.argpartition(x, (1, 3))]
-    array([1, 2, 3, 4])
-
-    """
-    return a.argpartition(kth, axis, kind=kind, order=order)
-
-
-def sort(a, axis=-1, kind='quicksort', order=None):
-    """
-    Return a sorted copy of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be sorted.
-    axis : int or None, optional
-        Axis along which to sort. If None, the array is flattened before
-        sorting. The default is -1, which sorts along the last axis.
-    kind : {'quicksort', 'mergesort', 'heapsort'}, optional
-        Sorting algorithm. Default is 'quicksort'.
-    order : list, optional
-        When `a` is a structured array, this argument specifies which fields
-        to compare first, second, and so on.  This list does not need to
-        include all of the fields.
-
-    Returns
-    -------
-    sorted_array : ndarray
-        Array of the same type and shape as `a`.
-
-    See Also
-    --------
-    ndarray.sort : Method to sort an array in-place.
-    argsort : Indirect sort.
-    lexsort : Indirect stable sort on multiple keys.
-    searchsorted : Find elements in a sorted array.
-    partition : Partial sort.
-
-    Notes
-    -----
-    The various sorting algorithms are characterized by their average speed,
-    worst case performance, work space size, and whether they are stable. A
-    stable sort keeps items with the same key in the same relative
-    order. The three available algorithms have the following
-    properties:
-
-    =========== ======= ============= ============ =======
-       kind      speed   worst case    work space  stable
-    =========== ======= ============= ============ =======
-    'quicksort'    1     O(n^2)            0          no
-    'mergesort'    2     O(n*log(n))      ~n/2        yes
-    'heapsort'     3     O(n*log(n))       0          no
-    =========== ======= ============= ============ =======
-
-    All the sort algorithms make temporary copies of the data when
-    sorting along any but the last axis.  Consequently, sorting along
-    the last axis is faster and uses less space than sorting along
-    any other axis.
-
-    The sort order for complex numbers is lexicographic. If both the real
-    and imaginary parts are non-nan then the order is determined by the
-    real parts except when they are equal, in which case the order is
-    determined by the imaginary parts.
-
-    Previous to numpy 1.4.0 sorting real and complex arrays containing nan
-    values led to undefined behaviour. In numpy versions >= 1.4.0 nan
-    values are sorted to the end. The extended sort order is:
-
-      * Real: [R, nan]
-      * Complex: [R + Rj, R + nanj, nan + Rj, nan + nanj]
-
-    where R is a non-nan real value. Complex values with the same nan
-    placements are sorted according to the non-nan part if it exists.
-    Non-nan values are sorted as before.
-
-    Examples
-    --------
-    >>> a = np.array([[1,4],[3,1]])
-    >>> np.sort(a)                # sort along the last axis
-    array([[1, 4],
-           [1, 3]])
-    >>> np.sort(a, axis=None)     # sort the flattened array
-    array([1, 1, 3, 4])
-    >>> np.sort(a, axis=0)        # sort along the first axis
-    array([[1, 1],
-           [3, 4]])
-
-    Use the `order` keyword to specify a field to use when sorting a
-    structured array:
-
-    >>> dtype = [('name', 'S10'), ('height', float), ('age', int)]
-    >>> values = [('Arthur', 1.8, 41), ('Lancelot', 1.9, 38),
-    ...           ('Galahad', 1.7, 38)]
-    >>> a = np.array(values, dtype=dtype)       # create a structured array
-    >>> np.sort(a, order='height')                        # doctest: +SKIP
-    array([('Galahad', 1.7, 38), ('Arthur', 1.8, 41),
-           ('Lancelot', 1.8999999999999999, 38)],
-          dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
-
-    Sort by age, then height if ages are equal:
-
-    >>> np.sort(a, order=['age', 'height'])               # doctest: +SKIP
-    array([('Galahad', 1.7, 38), ('Lancelot', 1.8999999999999999, 38),
-           ('Arthur', 1.8, 41)],
-          dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
-
-    """
-    if axis is None:
-        a = asanyarray(a).flatten()
-        axis = 0
-    else:
-        a = asanyarray(a).copy()
-    a.sort(axis, kind, order)
-    return a
-
-
-def argsort(a, axis=-1, kind='quicksort', order=None):
-    """
-    Returns the indices that would sort an array.
-
-    Perform an indirect sort along the given axis using the algorithm specified
-    by the `kind` keyword. It returns an array of indices of the same shape as
-    `a` that index data along the given axis in sorted order.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to sort.
-    axis : int or None, optional
-        Axis along which to sort.  The default is -1 (the last axis). If None,
-        the flattened array is used.
-    kind : {'quicksort', 'mergesort', 'heapsort'}, optional
-        Sorting algorithm.
-    order : list, optional
-        When `a` is an array with fields defined, this argument specifies
-        which fields to compare first, second, etc.  Not all fields need be
-        specified.
-
-    Returns
-    -------
-    index_array : ndarray, int
-        Array of indices that sort `a` along the specified axis.
-        In other words, ``a[index_array]`` yields a sorted `a`.
-
-    See Also
-    --------
-    sort : Describes sorting algorithms used.
-    lexsort : Indirect stable sort with multiple keys.
-    ndarray.sort : Inplace sort.
-    argpartition : Indirect partial sort.
-
-    Notes
-    -----
-    See `sort` for notes on the different sorting algorithms.
-
-    As of NumPy 1.4.0 `argsort` works with real/complex arrays containing
-    nan values. The enhanced sort order is documented in `sort`.
-
-    Examples
-    --------
-    One dimensional array:
-
-    >>> x = np.array([3, 1, 2])
-    >>> np.argsort(x)
-    array([1, 2, 0])
-
-    Two-dimensional array:
-
-    >>> x = np.array([[0, 3], [2, 2]])
-    >>> x
-    array([[0, 3],
-           [2, 2]])
-
-    >>> np.argsort(x, axis=0)
-    array([[0, 1],
-           [1, 0]])
-
-    >>> np.argsort(x, axis=1)
-    array([[0, 1],
-           [0, 1]])
-
-    Sorting with keys:
-
-    >>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '<i4'), ('y', '<i4')])
-    >>> x
-    array([(1, 0), (0, 1)],
-          dtype=[('x', '<i4'), ('y', '<i4')])
-
-    >>> np.argsort(x, order=('x','y'))
-    array([1, 0])
-
-    >>> np.argsort(x, order=('y','x'))
-    array([0, 1])
-
-    """
-    try:
-        argsort = a.argsort
-    except AttributeError:
-        return _wrapit(a, 'argsort', axis, kind, order)
-    return argsort(axis, kind, order)
-
-
-def argmax(a, axis=None):
-    """
-    Indices of the maximum values along an axis.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axis : int, optional
-        By default, the index is into the flattened array, otherwise
-        along the specified axis.
-
-    Returns
-    -------
-    index_array : ndarray of ints
-        Array of indices into the array. It has the same shape as `a.shape`
-        with the dimension along `axis` removed.
-
-    See Also
-    --------
-    ndarray.argmax, argmin
-    amax : The maximum value along a given axis.
-    unravel_index : Convert a flat index into an index tuple.
-
-    Notes
-    -----
-    In case of multiple occurrences of the maximum values, the indices
-    corresponding to the first occurrence are returned.
-
-    Examples
-    --------
-    >>> a = np.arange(6).reshape(2,3)
-    >>> a
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.argmax(a)
-    5
-    >>> np.argmax(a, axis=0)
-    array([1, 1, 1])
-    >>> np.argmax(a, axis=1)
-    array([2, 2])
-
-    >>> b = np.arange(6)
-    >>> b[1] = 5
-    >>> b
-    array([0, 5, 2, 3, 4, 5])
-    >>> np.argmax(b) # Only the first occurrence is returned.
-    1
-
-    """
-    assert axis is None
-    try:
-        argmax = a.argmax
-    except AttributeError:
-        return _wrapit(a, 'argmax')
-    return argmax()
-
-
-def argmin(a, axis=None):
-    """
-    Return the indices of the minimum values along an axis.
-
-    See Also
-    --------
-    argmax : Similar function.  Please refer to `numpy.argmax` for detailed
-        documentation.
-
-    """
-    assert axis is None
-    try:
-        argmin = a.argmin
-    except AttributeError:
-        return _wrapit(a, 'argmin')
-    return argmin()
-
-
-def searchsorted(a, v, side='left', sorter=None):
-    """
-    Find indices where elements should be inserted to maintain order.
-
-    Find the indices into a sorted array `a` such that, if the
-    corresponding elements in `v` were inserted before the indices, the
-    order of `a` would be preserved.
-
-    Parameters
-    ----------
-    a : 1-D array_like
-        Input array. If `sorter` is None, then it must be sorted in
-        ascending order, otherwise `sorter` must be an array of indices
-        that sort it.
-    v : array_like
-        Values to insert into `a`.
-    side : {'left', 'right'}, optional
-        If 'left', the index of the first suitable location found is given.
-        If 'right', return the last such index.  If there is no suitable
-        index, return either 0 or N (where N is the length of `a`).
-    sorter : 1-D array_like, optional
-        .. versionadded:: 1.7.0
-        Optional array of integer indices that sort array a into ascending
-        order. They are typically the result of argsort.
-
-    Returns
-    -------
-    indices : array of ints
-        Array of insertion points with the same shape as `v`.
-
-    See Also
-    --------
-    sort : Return a sorted copy of an array.
-    histogram : Produce histogram from 1-D data.
-
-    Notes
-    -----
-    Binary search is used to find the required insertion points.
-
-    As of Numpy 1.4.0 `searchsorted` works with real/complex arrays containing
-    `nan` values. The enhanced sort order is documented in `sort`.
-
-    Examples
-    --------
-    >>> np.searchsorted([1,2,3,4,5], 3)
-    2
-    >>> np.searchsorted([1,2,3,4,5], 3, side='right')
-    3
-    >>> np.searchsorted([1,2,3,4,5], [-10, 10, 2, 3])
-    array([0, 5, 1, 2])
-
-    """
-    try:
-        searchsorted = a.searchsorted
-    except AttributeError:
-        return _wrapit(a, 'searchsorted', v, side, sorter)
-    return searchsorted(v, side, sorter)
-
-
-def resize(a, new_shape):
-    """
-    Return a new array with the specified shape.
-
-    If the new array is larger than the original array, then the new
-    array is filled with repeated copies of `a`.  Note that this behavior
-    is different from a.resize(new_shape) which fills with zeros instead
-    of repeated copies of `a`.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be resized.
-
-    new_shape : int or tuple of int
-        Shape of resized array.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        The new array is formed from the data in the old array, repeated
-        if necessary to fill out the required number of elements.  The
-        data are repeated in the order that they are stored in memory.
-

From noreply at buildbot.pypy.org  Thu Oct 31 11:53:50 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 11:53:50 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Fix for tests: some tests use
	trace_eagerness=1
Message-ID: <20131031105350.63EF91C0112@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67763:1c80e5ce5fdc
Date: 2013-10-31 11:53 +0100
http://bitbucket.org/pypy/pypy/changeset/1c80e5ce5fdc/

Log:	Fix for tests: some tests use trace_eagerness=1

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -28,9 +28,7 @@
         """Return the 'increment' value corresponding to the given number."""
         if threshold <= 0:
             return 0.0   # no increment, never reach 1.0
-        if threshold < 2:
-            threshold = 2
-        return 1.0 / (threshold - 0.001)   # the number is at most 0.500xx
+        return 1.0 / (threshold - 0.001)
 
     def get_index(self, hash):
         """Return the index (< self.size) from a hash value.  This keeps
@@ -45,6 +43,8 @@
 
     def tick(self, index, increment):
         counter = float(self.timetable[index]) + increment
+        #print '-'*79
+        #print 'COUNTER TICK:', index, '-> %s' % counter
         if counter < 1.0:
             self.timetable[index] = r_singlefloat(counter)
             return False
@@ -53,6 +53,8 @@
     tick._always_inline_ = True
 
     def reset(self, index):
+        #print '-'*79
+        #print 'COUNTER RESET:', index
         self.timetable[index] = r_singlefloat(0.0)
 
     def lookup_chain(self, index):

From noreply at buildbot.pypy.org  Thu Oct 31 11:54:29 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 11:54:29 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Kill this test, now outdated
Message-ID: <20131031105429.C05891C0112@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67764:552b94e23692
Date: 2013-10-31 11:53 +0100
http://bitbucket.org/pypy/pypy/changeset/552b94e23692/

Log:	Kill this test, now outdated

diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -2,7 +2,6 @@
 from rpython.jit.metainterp.history import ConstInt, History, Stats
 from rpython.jit.metainterp.history import INT
 from rpython.jit.metainterp.compile import compile_loop
-from rpython.jit.metainterp.compile import ResumeGuardCountersInt
 from rpython.jit.metainterp.compile import compile_tmp_callback
 from rpython.jit.metainterp import jitexc
 from rpython.jit.metainterp import jitprof, typesystem, compile
@@ -100,54 +99,6 @@
     #
     del cpu.seen[:]
 
-def test_resume_guard_counters():
-    rgc = ResumeGuardCountersInt()
-    # fill in the table
-    for i in range(5):
-        count = rgc.see_int(100+i)
-        assert count == 1
-        count = rgc.see_int(100+i)
-        assert count == 2
-        assert rgc.counters == [0] * (4-i) + [2] * (1+i)
-    for i in range(5):
-        count = rgc.see_int(100+i)
-        assert count == 3
-    # make a distribution:  [5, 4, 7, 6, 3]
-    assert rgc.counters == [3, 3, 3, 3, 3]
-    count = rgc.see_int(101)
-    assert count == 4
-    count = rgc.see_int(101)
-    assert count == 5
-    count = rgc.see_int(101)
-    assert count == 6
-    count = rgc.see_int(102)
-    assert count == 4
-    count = rgc.see_int(102)
-    assert count == 5
-    count = rgc.see_int(102)
-    assert count == 6
-    count = rgc.see_int(102)
-    assert count == 7
-    count = rgc.see_int(103)
-    assert count == 4
-    count = rgc.see_int(104)
-    assert count == 4
-    count = rgc.see_int(104)
-    assert count == 5
-    assert rgc.counters == [5, 4, 7, 6, 3]
-    # the next new item should throw away 104, as 5 is the middle counter
-    count = rgc.see_int(190)
-    assert count == 1
-    assert rgc.counters == [1, 4, 7, 6, 3]
-    # the next new item should throw away 103, as 4 is the middle counter
-    count = rgc.see_int(191)
-    assert count == 1
-    assert rgc.counters == [1, 1, 7, 6, 3]
-    # the next new item should throw away 100, as 3 is the middle counter
-    count = rgc.see_int(192)
-    assert count == 1
-    assert rgc.counters == [1, 1, 7, 6, 1]
-
 
 def test_compile_tmp_callback():
     from rpython.jit.codewriter import heaptracker

From noreply at buildbot.pypy.org  Thu Oct 31 12:02:03 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 12:02:03 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Fix more tests
Message-ID: <20131031110203.7C6B41C0330@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67765:f8b0657d3fb4
Date: 2013-10-31 12:01 +0100
http://bitbucket.org/pypy/pypy/changeset/f8b0657d3fb4/

Log:	Fix more tests

diff --git a/rpython/jit/metainterp/test/test_memmgr.py b/rpython/jit/metainterp/test/test_memmgr.py
--- a/rpython/jit/metainterp/test/test_memmgr.py
+++ b/rpython/jit/metainterp/test/test_memmgr.py
@@ -15,7 +15,7 @@
 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.rlib.jit import JitDriver, dont_look_inside
 from rpython.jit.metainterp.warmspot import get_stats
-from rpython.jit.metainterp.warmstate import JitCell
+from rpython.jit.metainterp.warmstate import BaseJitCell
 from rpython.rlib import rgc
 
 class FakeLoopToken:
@@ -87,15 +87,15 @@
     # these tests to pass. But we dont want it there always since that will
     # make all other tests take forever.
     def setup_class(cls):
-        original_get_procedure_token = JitCell.get_procedure_token
+        original_get_procedure_token = BaseJitCell.get_procedure_token
         def get_procedure_token(self):
             rgc.collect();
             return original_get_procedure_token(self)
-        JitCell.get_procedure_token = get_procedure_token
+        BaseJitCell.get_procedure_token = get_procedure_token
         cls.original_get_procedure_token = original_get_procedure_token
 
     def teardown_class(cls):
-        JitCell.get_procedure_token = cls.original_get_procedure_token
+        BaseJitCell.get_procedure_token = cls.original_get_procedure_token
 
     def test_loop_kept_alive(self):
         myjitdriver = JitDriver(greens=[], reds=['n'])
diff --git a/rpython/jit/metainterp/test/test_warmstate.py b/rpython/jit/metainterp/test/test_warmstate.py
--- a/rpython/jit/metainterp/test/test_warmstate.py
+++ b/rpython/jit/metainterp/test/test_warmstate.py
@@ -3,10 +3,10 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.jit.metainterp.warmstate import wrap, unwrap, specialize_value
 from rpython.jit.metainterp.warmstate import equal_whatever, hash_whatever
-from rpython.jit.metainterp.warmstate import WarmEnterState, JitCell
-from rpython.jit.metainterp.warmstate import MODE_HAVE_PROC, MODE_TRACING
+from rpython.jit.metainterp.warmstate import WarmEnterState
 from rpython.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from rpython.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr
+from rpython.jit.metainterp.counter import DeterministicJitCounter
 from rpython.jit.codewriter import longlong
 from rpython.rlib.rarithmetic import r_singlefloat
 
@@ -78,69 +78,6 @@
     interpret(fn, [42], type_system='lltype')
 
 
-def test_make_jitcell_getter_default():
-    class FakeJitDriverSD:
-        _green_args_spec = [lltype.Signed, lltype.Float]
-    state = WarmEnterState(None, FakeJitDriverSD())
-    get_jitcell = state._make_jitcell_getter_default()
-    cell1 = get_jitcell(True, 42, 42.5)
-    assert isinstance(cell1, JitCell)
-    cell2 = get_jitcell(True, 42, 42.5)
-    assert cell1 is cell2
-    cell3 = get_jitcell(True, 41, 42.5)
-    assert get_jitcell(False, 42, 0.25) is None
-    cell4 = get_jitcell(True, 42, 0.25)
-    assert get_jitcell(False, 42, 0.25) is cell4
-    assert cell1 is not cell3 is not cell4 is not cell1
-
-def test_make_jitcell_getter():
-    class FakeJitDriverSD:
-        _green_args_spec = [lltype.Float]
-        _get_jitcell_at_ptr = None
-    state = WarmEnterState(None, FakeJitDriverSD())
-    get_jitcell = state.make_jitcell_getter()
-    cell1 = get_jitcell(True, 1.75)
-    cell2 = get_jitcell(True, 1.75)
-    assert cell1 is cell2
-    assert get_jitcell is state.make_jitcell_getter()
-
-def test_make_jitcell_getter_custom():
-    from rpython.rtyper.typesystem import LowLevelTypeSystem
-    class FakeRTyper:
-        type_system = LowLevelTypeSystem.instance
-    celldict = {}
-    def getter(x, y):
-        return celldict.get((x, y))
-    def setter(newcell, x, y):
-        newcell.x = x
-        newcell.y = y
-        celldict[x, y] = newcell
-    GETTER = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Float],
-                                        llmemory.GCREF))
-    SETTER = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed,
-                                         lltype.Float], lltype.Void))
-    class FakeWarmRunnerDesc:
-        rtyper = FakeRTyper()
-        cpu = None
-        memory_manager = None
-    class FakeJitDriverSD:
-        _get_jitcell_at_ptr = llhelper(GETTER, getter)
-        _set_jitcell_at_ptr = llhelper(SETTER, setter)
-    #
-    state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD())
-    get_jitcell = state._make_jitcell_getter_custom()
-    cell1 = get_jitcell(True, 5, 42.5)
-    assert isinstance(cell1, JitCell)
-    assert cell1.x == 5
-    assert cell1.y == 42.5
-    cell2 = get_jitcell(True, 5, 42.5)
-    assert cell2 is cell1
-    cell3 = get_jitcell(True, 41, 42.5)
-    assert get_jitcell(False, 42, 0.25) is None
-    cell4 = get_jitcell(True, 42, 0.25)
-    assert get_jitcell(False, 42, 0.25) is cell4
-    assert cell1 is not cell3 is not cell4 is not cell1
-
 def test_make_unwrap_greenkey():
     class FakeJitDriverSD:
         _green_args_spec = [lltype.Signed, lltype.Float]
@@ -150,26 +87,11 @@
     assert greenargs == (42, 42.5)
     assert type(greenargs[0]) is int
 
-def test_attach_unoptimized_bridge_from_interp():
-    class FakeJitDriverSD:
-        _green_args_spec = [lltype.Signed, lltype.Float]
-        _get_jitcell_at_ptr = None
-    state = WarmEnterState(None, FakeJitDriverSD())
-    get_jitcell = state.make_jitcell_getter()
-    class FakeLoopToken(object):
-        invalidated = False
-    looptoken = FakeLoopToken()
-    state.attach_procedure_to_interp([ConstInt(5),
-                                      constfloat(2.25)],
-                                     looptoken)
-    cell1 = get_jitcell(True, 5, 2.25)
-    assert cell1.mode == MODE_HAVE_PROC
-    assert cell1.get_procedure_token() is looptoken
-
 def test_make_jitdriver_callbacks_1():
     class FakeWarmRunnerDesc:
         cpu = None
         memory_manager = None
+        jitcounter = DeterministicJitCounter()
     class FakeJitDriverSD:
         jitdriver = None
         _green_args_spec = [lltype.Signed, lltype.Float]
@@ -199,13 +121,13 @@
         rtyper = None
         cpu = None
         memory_manager = None
+        jitcounter = DeterministicJitCounter()
     class FakeJitDriverSD:
         jitdriver = None
         _green_args_spec = [lltype.Signed, lltype.Float]
         _get_printable_location_ptr = llhelper(GET_LOCATION, get_location)
         _confirm_enter_jit_ptr = None
         _can_never_inline_ptr = None
-        _get_jitcell_at_ptr = None
         _should_unroll_one_iteration_ptr = None
         red_args_types = []
     state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD())
@@ -225,13 +147,13 @@
         rtyper = None
         cpu = None
         memory_manager = None
+        jitcounter = DeterministicJitCounter()
     class FakeJitDriverSD:
         jitdriver = None
         _green_args_spec = [lltype.Signed, lltype.Float]
         _get_printable_location_ptr = None
         _confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit)
         _can_never_inline_ptr = None
-        _get_jitcell_at_ptr = None
         _should_unroll_one_iteration_ptr = None
         red_args_types = []
 
@@ -251,13 +173,13 @@
         rtyper = None
         cpu = None
         memory_manager = None
+        jitcounter = DeterministicJitCounter()
     class FakeJitDriverSD:
         jitdriver = None
         _green_args_spec = [lltype.Signed, lltype.Float]
         _get_printable_location_ptr = None
         _confirm_enter_jit_ptr = None
         _can_never_inline_ptr = llhelper(CAN_NEVER_INLINE, can_never_inline)
-        _get_jitcell_at_ptr = None
         _should_unroll_one_iteration_ptr = None
         red_args_types = []
 
@@ -265,52 +187,3 @@
     state.make_jitdriver_callbacks()
     res = state.can_never_inline(5, 42.5)
     assert res is True
-
-def test_cleanup_jitcell_dict():
-    class FakeJitDriverSD:
-        _green_args_spec = [lltype.Signed]
-    #
-    # Test creating tons of jitcells that remain at 0
-    warmstate = WarmEnterState(None, FakeJitDriverSD())
-    get_jitcell = warmstate._make_jitcell_getter_default()
-    cell1 = get_jitcell(True, -1)
-    assert len(warmstate._jitcell_dict) == 1
-    #
-    for i in range(1, 20005):
-        get_jitcell(True, i)     # should trigger a clean-up at 20001
-        assert len(warmstate._jitcell_dict) == (i % 20000) + 1
-    #
-    # Same test, with one jitcell that has a counter of BASE instead of 0
-    warmstate = WarmEnterState(None, FakeJitDriverSD())
-    get_jitcell = warmstate._make_jitcell_getter_default()
-    cell2 = get_jitcell(True, -2)
-    cell2.counter = BASE = warmstate.THRESHOLD_LIMIT // 2    # 50%
-    #
-    for i in range(0, 20005):
-        get_jitcell(True, i)
-        assert len(warmstate._jitcell_dict) == (i % 19999) + 2
-    #
-    assert cell2 in warmstate._jitcell_dict.values()
-    assert cell2.counter == int(BASE * 0.92)   # decayed once
-    #
-    # Same test, with jitcells that are compiled and freed by the memmgr
-    warmstate = WarmEnterState(None, FakeJitDriverSD())
-    get_jitcell = warmstate._make_jitcell_getter_default()
-    get_jitcell(True, -1)
-    #
-    for i in range(1, 20005):
-        cell = get_jitcell(True, i)
-        cell.mode = MODE_HAVE_PROC
-        cell.wref_procedure_token = None    # or a dead weakref, equivalently
-        assert len(warmstate._jitcell_dict) == (i % 20000) + 1
-    #
-    # Same test, with mode == MODE_TRACING (rare case, kept alive)
-    warmstate = WarmEnterState(None, FakeJitDriverSD())
-    get_jitcell = warmstate._make_jitcell_getter_default()
-    cell = get_jitcell(True, -1)
-    cell.mode = MODE_TRACING
-    #
-    for i in range(1, 20005):
-        cell = get_jitcell(True, i)
-        cell.mode = MODE_TRACING
-        assert len(warmstate._jitcell_dict) == i + 1
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -174,9 +174,10 @@
             self.profiler = None
         # initialize the state with the default values of the
         # parameters specified in rlib/jit.py
-        for name, default_value in PARAMETERS.items():
-            meth = getattr(self, 'set_param_' + name)
-            meth(default_value)
+        if self.warmrunnerdesc is not None:
+            for name, default_value in PARAMETERS.items():
+                meth = getattr(self, 'set_param_' + name)
+                meth(default_value)
 
     def _compute_threshold(self, threshold):
         return self.warmrunnerdesc.jitcounter.compute_threshold(threshold)

From noreply at buildbot.pypy.org  Thu Oct 31 12:12:01 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 12:12:01 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Fix: the hash was miscomputed
Message-ID: <20131031111201.F0E3B1C0330@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67766:078369425c4e
Date: 2013-10-31 12:11 +0100
http://bitbucket.org/pypy/pypy/changeset/078369425c4e/

Log:	Fix: the hash was miscomputed

diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -411,7 +411,7 @@
         #
         jitcounter = self.warmrunnerdesc.jitcounter
         jitdriver_sd = self.jitdriver_sd
-        green_args_spec = unrolling_iterable([('g%d' % i, TYPE)
+        green_args_name_spec = unrolling_iterable([('g%d' % i, TYPE)
                      for i, TYPE in enumerate(jitdriver_sd._green_args_spec)])
         unwrap_greenkey = self.make_unwrap_greenkey()
         random_initial_value = hash(self)
@@ -419,13 +419,13 @@
         class JitCell(BaseJitCell):
             def __init__(self, *greenargs):
                 i = 0
-                for attrname, _ in green_args_spec:
+                for attrname, _ in green_args_name_spec:
                     setattr(self, attrname, greenargs[i])
                     i = i + 1
 
             def comparekey(self, *greenargs2):
                 i = 0
-                for attrname, TYPE in green_args_spec:
+                for attrname, TYPE in green_args_name_spec:
                     item1 = getattr(self, attrname)
                     if not equal_whatever(TYPE, item1, greenargs2[i]):
                         return False
@@ -436,7 +436,7 @@
             def get_index(*greenargs):
                 x = random_initial_value
                 i = 0
-                for TYPE in green_args_spec:
+                for _, TYPE in green_args_name_spec:
                     item = greenargs[i]
                     y = hash_whatever(TYPE, item)
                     x = intmask((x ^ y) * 1405695061)  # prime number, 2**30~31

From noreply at buildbot.pypy.org  Thu Oct 31 12:38:41 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 12:38:41 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Fix for guard counters,
	more test fixes
Message-ID: <20131031113841.E53811C0330@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67767:f2bdf363205b
Date: 2013-10-31 12:21 +0100
http://bitbucket.org/pypy/pypy/changeset/f2bdf363205b/

Log:	Fix for guard counters, more test fixes

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -43,18 +43,16 @@
 
     def tick(self, index, increment):
         counter = float(self.timetable[index]) + increment
-        #print '-'*79
-        #print 'COUNTER TICK:', index, '-> %s' % counter
         if counter < 1.0:
             self.timetable[index] = r_singlefloat(counter)
             return False
         else:
+            # when the bound is reached, we immediately reset the value to 0.0
+            self.reset(index)
             return True
     tick._always_inline_ = True
 
     def reset(self, index):
-        #print '-'*79
-        #print 'COUNTER RESET:', index
         self.timetable[index] = r_singlefloat(0.0)
 
     def lookup_chain(self, index):
diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py
--- a/rpython/jit/metainterp/test/test_recursive.py
+++ b/rpython/jit/metainterp/test/test_recursive.py
@@ -342,7 +342,7 @@
         assert res == 0
         self.check_max_trace_length(TRACE_LIMIT)
         self.check_enter_count_at_most(10) # maybe
-        self.check_aborted_count(7)
+        self.check_aborted_count(6)
 
     def test_trace_limit_bridge(self):
         def recursive(n):
@@ -425,7 +425,7 @@
 
         res = self.meta_interp(loop, [20], failargs_limit=FAILARGS_LIMIT,
                                listops=True)
-        self.check_aborted_count(5)
+        self.check_aborted_count(4)
 
     def test_max_failure_args_exc(self):
         FAILARGS_LIMIT = 10
@@ -465,7 +465,7 @@
         res = self.meta_interp(main, [20], failargs_limit=FAILARGS_LIMIT,
                                listops=True)
         assert not res
-        self.check_aborted_count(5)
+        self.check_aborted_count(4)
 
     def test_set_param_inlining(self):
         myjitdriver = JitDriver(greens=[], reds=['n', 'recurse'])
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -312,7 +312,6 @@
             assert 0, "should have raised"
 
         def bound_reached(index, *args):
-            jitcounter.reset(index)
             if not confirm_enter_jit(*args):
                 return
             # start tracing

From noreply at buildbot.pypy.org  Thu Oct 31 12:38:43 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 12:38:43 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Fix: if we have a
 compile_tmp_callback in a JitCell, use the jitcounter anyway
Message-ID: <20131031113843.1CAA11C0330@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67768:81f6d24ea133
Date: 2013-10-31 12:38 +0100
http://bitbucket.org/pypy/pypy/changeset/81f6d24ea133/

Log:	Fix: if we have a compile_tmp_callback in a JitCell, use the
	jitcounter anyway to know when we have to really compile it.

diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -126,6 +126,7 @@
 
 class BaseJitCell(object):
     tracing = False
+    temporary = False
     dont_trace_here = chr(0)
     wref_procedure_token = None
     next = None
@@ -137,8 +138,9 @@
                 return token
         return None
 
-    def set_procedure_token(self, token):
+    def set_procedure_token(self, token, tmp=False):
         self.wref_procedure_token = self._makeref(token)
+        self.temporary = tmp
 
     def _makeref(self, token):
         assert token is not None
@@ -311,20 +313,21 @@
             #
             assert 0, "should have raised"
 
-        def bound_reached(index, *args):
+        def bound_reached(index, cell, *args):
             if not confirm_enter_jit(*args):
                 return
             # start tracing
             from rpython.jit.metainterp.pyjitpl import MetaInterp
             metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
             greenargs = args[:num_green_args]
-            newcell = JitCell(*greenargs)
-            newcell.tracing = True
-            jitcounter.install_new_cell(index, newcell)
+            if cell is None:
+                cell = JitCell(*greenargs)
+                jitcounter.install_new_cell(index, cell)
+            cell.tracing = True
             try:
                 metainterp.compile_and_run_once(jitdriver_sd, *args)
             finally:
-                newcell.tracing = False
+                cell.tracing = False
 
         def maybe_compile_and_run(increment_threshold, *args):
             """Entry point to the JIT.  Called at the point with the
@@ -344,7 +347,7 @@
             else:
                 # not found. increment the counter
                 if jitcounter.tick(index, increment_threshold):
-                    bound_reached(index, *args)
+                    bound_reached(index, None, *args)
                 return
 
             # Here, we have found 'cell'.
@@ -353,6 +356,11 @@
                 # tracing already happening in some outer invocation of
                 # this function. don't trace a second time.
                 return
+            if cell.temporary:
+                # attached by compile_tmp_callback().  count normally
+                if jitcounter.tick(index, increment_threshold):
+                    bound_reached(index, cell, *args)
+                return
             # machine code was already compiled for these greenargs
             procedure_token = cell.get_procedure_token()
             if procedure_token is None:
@@ -517,15 +525,10 @@
             procedure_token = cell.get_procedure_token()
             if procedure_token is None:
                 from rpython.jit.metainterp.compile import compile_tmp_callback
-                if cell.mode == MODE_HAVE_PROC:
-                    # used to be a valid entry bridge,
-                    # but was freed in the meantime.
-                    cell.counter = 0
-                    cell.mode = MODE_COUNTING
                 memmgr = warmrunnerdesc.memory_manager
                 procedure_token = compile_tmp_callback(cpu, jd, greenkey,
                                                        redargtypes, memmgr)
-                cell.set_procedure_token(procedure_token)
+                cell.set_procedure_token(procedure_token, tmp=True)
             return procedure_token
         self.get_assembler_token = get_assembler_token
 

From noreply at buildbot.pypy.org  Thu Oct 31 12:44:09 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 12:44:09 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Test fix: force turning
 AddressAsInt into real ints for hashing
Message-ID: <20131031114409.19A361C0112@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67769:14343c11219f
Date: 2013-10-31 12:43 +0100
http://bitbucket.org/pypy/pypy/changeset/14343c11219f/

Log:	Test fix: force turning AddressAsInt into real ints for hashing

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1,5 +1,5 @@
 import weakref
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
@@ -590,6 +590,11 @@
             else:
                 assert 0, typetag
 
+            if not we_are_translated():
+                if isinstance(intval, llmemory.AddressAsInt):
+                    intval = llmemory.cast_adr_to_int(
+                        llmemory.cast_int_to_adr(intval), "forced")
+
             hash = (current_object_addr_as_int(self) * 777767777 +
                     intval * 1442968193)
             index = jitcounter.get_index(hash)

From noreply at buildbot.pypy.org  Thu Oct 31 12:58:26 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 12:58:26 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Test fix
Message-ID: <20131031115826.1613E1C0F88@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67770:aad0c3f309c9
Date: 2013-10-31 12:48 +0100
http://bitbucket.org/pypy/pypy/changeset/aad0c3f309c9/

Log:	Test fix

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -16,6 +16,7 @@
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
 from rpython.jit.metainterp import compile, resume, history
 from rpython.jit.metainterp.jitprof import EmptyProfiler
+from rpython.jit.metainterp.counter import DeterministicJitCounter
 from rpython.config.translationoption import get_combined_translation_config
 from rpython.jit.metainterp.resoperation import rop, opname, ResOperation
 from rpython.jit.metainterp.optimizeopt.unroll import Inliner
@@ -306,6 +307,7 @@
         class memory_manager:
             retrace_limit = 5
             max_retrace_guards = 15
+        jitcounter = DeterministicJitCounter()
 
 class Storage(compile.ResumeGuardDescr):
     "for tests."

From noreply at buildbot.pypy.org  Thu Oct 31 12:58:27 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 12:58:27 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Fix test
Message-ID: <20131031115827.3D5441C0F88@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67771:9b42bd9dfbe7
Date: 2013-10-31 12:50 +0100
http://bitbucket.org/pypy/pypy/changeset/9b42bd9dfbe7/

Log:	Fix test

diff --git a/rpython/jit/metainterp/test/test_counter.py b/rpython/jit/metainterp/test/test_counter.py
--- a/rpython/jit/metainterp/test/test_counter.py
+++ b/rpython/jit/metainterp/test/test_counter.py
@@ -18,18 +18,16 @@
     incr = jc.compute_threshold(4)
     for i in range(5):
         r = jc.tick(104, incr)
-        assert r is (i >= 3)
+        assert r is (i == 3)
     for i in range(5):
         r = jc.tick(108, incr)
         s = jc.tick(109, incr)
-        assert r is (i >= 3)
-        assert s is (i >= 3)
+        assert r is (i == 3)
+        assert s is (i == 3)
     jc.reset(108)
     for i in range(5):
         r = jc.tick(108, incr)
-        s = jc.tick(109, incr)
-        assert r is (i >= 3)
-        assert s is True
+        assert r is (i == 3)
 
 def test_install_new_chain():
     class Dead:

From noreply at buildbot.pypy.org  Thu Oct 31 12:58:28 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 12:58:28 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Translation fix
Message-ID: <20131031115828.77D891C0F88@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67772:5a1e8921b8f3
Date: 2013-10-31 12:51 +0100
http://bitbucket.org/pypy/pypy/changeset/5a1e8921b8f3/

Log:	Translation fix

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -1,4 +1,4 @@
-from rpython.rlib.rarithmetic import r_singlefloat, intmask
+from rpython.rlib.rarithmetic import r_singlefloat, intmask, r_uint
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
@@ -33,7 +33,7 @@
     def get_index(self, hash):
         """Return the index (< self.size) from a hash value.  This keeps
         the *high* bits of hash!  Be sure that hash is computed correctly."""
-        return intmask(r_uint32(hash) >> self.shift)
+        return intmask(r_uint32(r_uint(hash) >> self.shift))
     get_index._always_inline_ = True
 
     def fetch_next_index(self):

From noreply at buildbot.pypy.org  Thu Oct 31 12:58:29 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 12:58:29 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Expand the docstring
Message-ID: <20131031115829.AF9751C0F88@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67773:42b5b4e34f4d
Date: 2013-10-31 12:53 +0100
http://bitbucket.org/pypy/pypy/changeset/42b5b4e34f4d/

Log:	Expand the docstring

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -31,8 +31,9 @@
         return 1.0 / (threshold - 0.001)
 
     def get_index(self, hash):
-        """Return the index (< self.size) from a hash value.  This keeps
-        the *high* bits of hash!  Be sure that hash is computed correctly."""
+        """Return the index (< self.size) from a hash value.  This truncates
+        the hash to 32 bits, and then keep the *highest* remaining bits.
+        Be sure that hash is computed correctly."""
         return intmask(r_uint32(r_uint(hash) >> self.shift))
     get_index._always_inline_ = True
 

From noreply at buildbot.pypy.org  Thu Oct 31 12:58:31 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 12:58:31 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: hg merge default
Message-ID: <20131031115831.41DDB1C0F88@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67774:96c215655c56
Date: 2013-10-31 12:54 +0100
http://bitbucket.org/pypy/pypy/changeset/96c215655c56/

Log:	hg merge default

diff too long, truncating to 2000 out of 7885 lines

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -1,5 +1,5 @@
 =====================================
-PyPy: Python in Python Implementation 
+PyPy: Python in Python Implementation
 =====================================
 
 Welcome to PyPy!
@@ -26,9 +26,11 @@
 Building
 ========
 
-build with::
+build with:
 
-  rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+.. code-block:: console
+
+    $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
 
 This ends up with ``pypy-c`` binary in the main pypy directory. We suggest
 to use virtualenv with the resulting pypy-c as the interpreter, you can
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from . import core
-from .core import *
-from . import lib
-from .lib import *
-
-from __builtin__ import bool, int, long, float, complex, object, unicode, str
-
-from .core import round, abs, max, min
-
-__version__ = '1.7.0'
-
-__all__ = ['__version__']
-__all__ += core.__all__
-__all__ += lib.__all__
-
-#import sys
-#sys.modules.setdefault('numpy', sys.modules['numpypy'])
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from __future__ import division, absolute_import, print_function
-
-from . import multiarray
-from . import umath
-from . import numeric
-from .numeric import *
-from . import fromnumeric
-from .fromnumeric import *
-from . import shape_base
-from .shape_base import *
-
-from .fromnumeric import amax as max, amin as min, \
-    round_ as round
-from .numeric import absolute as abs
-
-__all__ = []
-__all__ += numeric.__all__
-__all__ += fromnumeric.__all__
-__all__ += shape_base.__all__
diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/_methods.py
+++ /dev/null
@@ -1,124 +0,0 @@
-"""
-Array methods which are called by the both the C-code for the method
-and the Python code for the NumPy-namespace function
-
-"""
-from __future__ import division, absolute_import, print_function
-
-import warnings
-
-from . import multiarray as mu
-from . import umath as um
-from .numeric import asanyarray
-from . import numerictypes as nt
-
-def _amax(a, axis=None, out=None, keepdims=False):
-    return um.maximum.reduce(a, axis=axis,
-                            out=out, keepdims=keepdims)
-
-def _amin(a, axis=None, out=None, keepdims=False):
-    return um.minimum.reduce(a, axis=axis,
-                            out=out, keepdims=keepdims)
-
-def _sum(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.add.reduce(a, axis=axis, dtype=dtype,
-                            out=out, keepdims=keepdims)
-
-def _prod(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.multiply.reduce(a, axis=axis, dtype=dtype,
-                            out=out, keepdims=keepdims)
-
-def _any(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.logical_or.reduce(a, axis=axis, dtype=dtype, out=out,
-                                keepdims=keepdims)
-
-def _all(a, axis=None, dtype=None, out=None, keepdims=False):
-    return um.logical_and.reduce(a, axis=axis, dtype=dtype, out=out,
-                                 keepdims=keepdims)
-
-def _count_reduce_items(arr, axis):
-    if axis is None:
-        axis = tuple(range(arr.ndim))
-    if not isinstance(axis, tuple):
-        axis = (axis,)
-    items = 1
-    for ax in axis:
-        items *= arr.shape[ax]
-    return items
-
-def _mean(a, axis=None, dtype=None, out=None, keepdims=False):
-    arr = asanyarray(a)
-
-    rcount = _count_reduce_items(arr, axis)
-    # Make this warning show up first
-    if rcount == 0:
-        warnings.warn("Mean of empty slice.", RuntimeWarning)
-
-
-    # Cast bool, unsigned int, and int to float64 by default
-    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
-        dtype = mu.dtype('f8')
-
-    ret = um.add.reduce(arr, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
-    if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(
-                ret, rcount, out=ret, casting='unsafe', subok=False)
-    else:
-        ret = ret.dtype.type(ret / rcount)
-
-    return ret
-
-def _var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
-    arr = asanyarray(a)
-
-    rcount = _count_reduce_items(arr, axis)
-    # Make this warning show up on top.
-    if ddof >= rcount:
-        warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning)
-
-    # Cast bool, unsigned int, and int to float64 by default
-    if dtype is None and issubclass(arr.dtype.type, (nt.integer, nt.bool_)):
-        dtype = mu.dtype('f8')
-
-    # Compute the mean.
-    # Note that if dtype is not of inexact type then arraymean will
-    # not be either.
-    arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True)
-    if isinstance(arrmean, mu.ndarray):
-        arrmean = um.true_divide(
-                arrmean, rcount, out=arrmean, casting='unsafe', subok=False)
-    else:
-        arrmean = arrmean.dtype.type(arrmean / rcount)
-
-    # Compute sum of squared deviations from mean
-    # Note that x may not be inexact and that we need it to be an array,
-    # not a scalar.
-    x = asanyarray(arr - arrmean)
-    if issubclass(arr.dtype.type, nt.complexfloating):
-        x = um.multiply(x, um.conjugate(x), out=x).real
-    else:
-        x = um.multiply(x, x, out=x)
-    ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
-
-    # Compute degrees of freedom and make sure it is not negative.
-    rcount = max([rcount - ddof, 0])
-
-    # divide by degrees of freedom
-    if isinstance(ret, mu.ndarray):
-        ret = um.true_divide(
-                ret, rcount, out=ret, casting='unsafe', subok=False)
-    else:
-        ret = ret.dtype.type(ret / rcount)
-
-    return ret
-
-def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
-    ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
-               keepdims=keepdims)
-
-    if isinstance(ret, mu.ndarray):
-        ret = um.sqrt(ret, out=ret)
-    else:
-        ret = ret.dtype.type(um.sqrt(ret))
-
-    return ret
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/arrayprint.py
+++ /dev/null
@@ -1,751 +0,0 @@
-"""Array printing function
-
-$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $
-"""
-__all__ = ["array2string", "set_printoptions", "get_printoptions"]
-__docformat__ = 'restructuredtext'
-
-#
-# Written by Konrad Hinsen <hinsenk at ere.umontreal.ca>
-# last revision: 1996-3-13
-# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details)
-# and by Perry Greenfield 2000-4-1 for numarray
-# and by Travis Oliphant  2005-8-22 for numpy
-
-import sys
-import numerictypes as _nt
-from umath import maximum, minimum, absolute, not_equal, isnan, isinf
-#from multiarray import format_longfloat, datetime_as_string, datetime_data
-from fromnumeric import ravel
-
-
-def product(x, y): return x*y
-
-_summaryEdgeItems = 3     # repr N leading and trailing items of each dimension
-_summaryThreshold = 1000 # total items > triggers array summarization
-
-_float_output_precision = 8
-_float_output_suppress_small = False
-_line_width = 75
-_nan_str = 'nan'
-_inf_str = 'inf'
-_formatter = None  # formatting function for array elements
-
-if sys.version_info[0] >= 3:
-    from functools import reduce
-
-def set_printoptions(precision=None, threshold=None, edgeitems=None,
-                     linewidth=None, suppress=None,
-                     nanstr=None, infstr=None,
-                     formatter=None):
-    """
-    Set printing options.
-
-    These options determine the way floating point numbers, arrays and
-    other NumPy objects are displayed.
-
-    Parameters
-    ----------
-    precision : int, optional
-        Number of digits of precision for floating point output (default 8).
-    threshold : int, optional
-        Total number of array elements which trigger summarization
-        rather than full repr (default 1000).
-    edgeitems : int, optional
-        Number of array items in summary at beginning and end of
-        each dimension (default 3).
-    linewidth : int, optional
-        The number of characters per line for the purpose of inserting
-        line breaks (default 75).
-    suppress : bool, optional
-        Whether or not suppress printing of small floating point values
-        using scientific notation (default False).
-    nanstr : str, optional
-        String representation of floating point not-a-number (default nan).
-    infstr : str, optional
-        String representation of floating point infinity (default inf).
-    formatter : dict of callables, optional
-        If not None, the keys should indicate the type(s) that the respective
-        formatting function applies to.  Callables should return a string.
-        Types that are not specified (by their corresponding keys) are handled
-        by the default formatters.  Individual types for which a formatter
-        can be set are::
-
-            - 'bool'
-            - 'int'
-            - 'timedelta' : a `numpy.timedelta64`
-            - 'datetime' : a `numpy.datetime64`
-            - 'float'
-            - 'longfloat' : 128-bit floats
-            - 'complexfloat'
-            - 'longcomplexfloat' : composed of two 128-bit floats
-            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
-            - 'str' : all other strings
-
-        Other keys that can be used to set a group of types at once are::
-
-            - 'all' : sets all types
-            - 'int_kind' : sets 'int'
-            - 'float_kind' : sets 'float' and 'longfloat'
-            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
-            - 'str_kind' : sets 'str' and 'numpystr'
-
-    See Also
-    --------
-    get_printoptions, set_string_function, array2string
-
-    Notes
-    -----
-    `formatter` is always reset with a call to `set_printoptions`.
-
-    Examples
-    --------
-    Floating point precision can be set:
-
-    >>> np.set_printoptions(precision=4)
-    >>> print np.array([1.123456789])
-    [ 1.1235]
-
-    Long arrays can be summarised:
-
-    >>> np.set_printoptions(threshold=5)
-    >>> print np.arange(10)
-    [0 1 2 ..., 7 8 9]
-
-    Small results can be suppressed:
-
-    >>> eps = np.finfo(float).eps
-    >>> x = np.arange(4.)
-    >>> x**2 - (x + eps)**2
-    array([ -4.9304e-32,  -4.4409e-16,   0.0000e+00,   0.0000e+00])
-    >>> np.set_printoptions(suppress=True)
-    >>> x**2 - (x + eps)**2
-    array([-0., -0.,  0.,  0.])
-
-    A custom formatter can be used to display array elements as desired:
-
-    >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)})
-    >>> x = np.arange(3)
-    >>> x
-    array([int: 0, int: -1, int: -2])
-    >>> np.set_printoptions()  # formatter gets reset
-    >>> x
-    array([0, 1, 2])
-
-    To put back the default options, you can use:
-
-    >>> np.set_printoptions(edgeitems=3,infstr='inf',
-    ... linewidth=75, nanstr='nan', precision=8,
-    ... suppress=False, threshold=1000, formatter=None)
-    """
-
-    global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \
-           _line_width, _float_output_suppress_small, _nan_str, _inf_str, \
-           _formatter
-    if linewidth is not None:
-        _line_width = linewidth
-    if threshold is not None:
-        _summaryThreshold = threshold
-    if edgeitems is not None:
-        _summaryEdgeItems = edgeitems
-    if precision is not None:
-        _float_output_precision = precision
-    if suppress is not None:
-        _float_output_suppress_small = not not suppress
-    if nanstr is not None:
-        _nan_str = nanstr
-    if infstr is not None:
-        _inf_str = infstr
-    _formatter = formatter
-
-def get_printoptions():
-    """
-    Return the current print options.
-
-    Returns
-    -------
-    print_opts : dict
-        Dictionary of current print options with keys
-
-          - precision : int
-          - threshold : int
-          - edgeitems : int
-          - linewidth : int
-          - suppress : bool
-          - nanstr : str
-          - infstr : str
-          - formatter : dict of callables
-
-        For a full description of these options, see `set_printoptions`.
-
-    See Also
-    --------
-    set_printoptions, set_string_function
-
-    """
-    d = dict(precision=_float_output_precision,
-             threshold=_summaryThreshold,
-             edgeitems=_summaryEdgeItems,
-             linewidth=_line_width,
-             suppress=_float_output_suppress_small,
-             nanstr=_nan_str,
-             infstr=_inf_str,
-             formatter=_formatter)
-    return d
-
-def _leading_trailing(a):
-    import numeric as _nc
-    if a.ndim == 1:
-        if len(a) > 2*_summaryEdgeItems:
-            b = _nc.concatenate((a[:_summaryEdgeItems],
-                                     a[-_summaryEdgeItems:]))
-        else:
-            b = a
-    else:
-        if len(a) > 2*_summaryEdgeItems:
-            l = [_leading_trailing(a[i]) for i in range(
-                min(len(a), _summaryEdgeItems))]
-            l.extend([_leading_trailing(a[-i]) for i in range(
-                min(len(a), _summaryEdgeItems),0,-1)])
-        else:
-            l = [_leading_trailing(a[i]) for i in range(0, len(a))]
-        b = _nc.concatenate(tuple(l))
-    return b
-
-def _boolFormatter(x):
-    if x:
-        return ' True'
-    else:
-        return 'False'
-
-
-def repr_format(x):
-    return repr(x)
-
-def _array2string(a, max_line_width, precision, suppress_small, separator=' ',
-                  prefix="", formatter=None):
-
-    if max_line_width is None:
-        max_line_width = _line_width
-
-    if precision is None:
-        precision = _float_output_precision
-
-    if suppress_small is None:
-        suppress_small = _float_output_suppress_small
-
-    if formatter is None:
-        formatter = _formatter
-
-    if a.size > _summaryThreshold:
-        summary_insert = "..., "
-        data = _leading_trailing(a)
-    else:
-        summary_insert = ""
-        data = ravel(a)
-
-    formatdict = {'bool' : _boolFormatter,
-                  'int' : IntegerFormat(data),
-                  'float' : FloatFormat(data, precision, suppress_small),
-                  'longfloat' : FloatFormat(data, precision, suppress_small),
-                  'complexfloat' : ComplexFormat(data, precision,
-                                                 suppress_small),
-                  'longcomplexfloat' : ComplexFormat(data, precision,
-                                                     suppress_small),
-                  'datetime' : DatetimeFormat(data),
-                  'timedelta' : TimedeltaFormat(data),
-                  'numpystr' : repr_format,
-                  'str' : str}
-
-    if formatter is not None:
-        fkeys = [k for k in formatter.keys() if formatter[k] is not None]
-        if 'all' in fkeys:
-            for key in formatdict.keys():
-                formatdict[key] = formatter['all']
-        if 'int_kind' in fkeys:
-            for key in ['int']:
-                formatdict[key] = formatter['int_kind']
-        if 'float_kind' in fkeys:
-            for key in ['float', 'longfloat']:
-                formatdict[key] = formatter['float_kind']
-        if 'complex_kind' in fkeys:
-            for key in ['complexfloat', 'longcomplexfloat']:
-                formatdict[key] = formatter['complex_kind']
-        if 'str_kind' in fkeys:
-            for key in ['numpystr', 'str']:
-                formatdict[key] = formatter['str_kind']
-        for key in formatdict.keys():
-            if key in fkeys:
-                formatdict[key] = formatter[key]
-
-    try:
-        format_function = a._format
-        msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \
-              "will be removed in 2.1. Use the `formatter` kw instead."
-        import warnings
-        warnings.warn(msg, DeprecationWarning)
-    except AttributeError:
-        # find the right formatting function for the array
-        dtypeobj = a.dtype.type
-        if issubclass(dtypeobj, _nt.bool_):
-            format_function = formatdict['bool']
-        elif issubclass(dtypeobj, _nt.integer):
-            #if issubclass(dtypeobj, _nt.timedelta64):
-            #    format_function = formatdict['timedelta']
-            #else:
-            format_function = formatdict['int']
-        elif issubclass(dtypeobj, _nt.floating):
-            if hasattr(_nt, 'longfloat') and issubclass(dtypeobj, _nt.longfloat):
-                format_function = formatdict['longfloat']
-            else:
-                format_function = formatdict['float']
-        elif issubclass(dtypeobj, _nt.complexfloating):
-            if hasattr(_nt, 'clongfloat') and issubclass(dtypeobj, _nt.clongfloat):
-                format_function = formatdict['longcomplexfloat']
-            else:
-                format_function = formatdict['complexfloat']
-        elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)):
-            format_function = formatdict['numpystr']
-        #elif issubclass(dtypeobj, _nt.datetime64):
-        #    format_function = formatdict['datetime']
-        else:
-            format_function = formatdict['str']
-
-    # skip over "["
-    next_line_prefix = " "
-    # skip over array(
-    next_line_prefix += " "*len(prefix)
-
-    lst = _formatArray(a, format_function, len(a.shape), max_line_width,
-                       next_line_prefix, separator,
-                       _summaryEdgeItems, summary_insert)[:-1]
-    return lst
-
-def _convert_arrays(obj):
-    import numeric as _nc
-    newtup = []
-    for k in obj:
-        if isinstance(k, _nc.ndarray):
-            k = k.tolist()
-        elif isinstance(k, tuple):
-            k = _convert_arrays(k)
-        newtup.append(k)
-    return tuple(newtup)
-
-
-def array2string(a, max_line_width=None, precision=None,
-                 suppress_small=None, separator=' ', prefix="",
-                 style=repr, formatter=None):
-    """
-    Return a string representation of an array.
-
-    Parameters
-    ----------
-    a : ndarray
-        Input array.
-    max_line_width : int, optional
-        The maximum number of columns the string should span. Newline
-        characters splits the string appropriately after array elements.
-    precision : int, optional
-        Floating point precision. Default is the current printing
-        precision (usually 8), which can be altered using `set_printoptions`.
-    suppress_small : bool, optional
-        Represent very small numbers as zero. A number is "very small" if it
-        is smaller than the current printing precision.
-    separator : str, optional
-        Inserted between elements.
-    prefix : str, optional
-        An array is typically printed as::
-
-          'prefix(' + array2string(a) + ')'
-
-        The length of the prefix string is used to align the
-        output correctly.
-    style : function, optional
-        A function that accepts an ndarray and returns a string.  Used only
-        when the shape of `a` is equal to ``()``, i.e. for 0-D arrays.
-    formatter : dict of callables, optional
-        If not None, the keys should indicate the type(s) that the respective
-        formatting function applies to.  Callables should return a string.
-        Types that are not specified (by their corresponding keys) are handled
-        by the default formatters.  Individual types for which a formatter
-        can be set are::
-
-            - 'bool'
-            - 'int'
-            - 'timedelta' : a `numpy.timedelta64`
-            - 'datetime' : a `numpy.datetime64`
-            - 'float'
-            - 'longfloat' : 128-bit floats
-            - 'complexfloat'
-            - 'longcomplexfloat' : composed of two 128-bit floats
-            - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
-            - 'str' : all other strings
-
-        Other keys that can be used to set a group of types at once are::
-
-            - 'all' : sets all types
-            - 'int_kind' : sets 'int'
-            - 'float_kind' : sets 'float' and 'longfloat'
-            - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
-            - 'str_kind' : sets 'str' and 'numpystr'
-
-    Returns
-    -------
-    array_str : str
-        String representation of the array.
-
-    Raises
-    ------
-    TypeError : if a callable in `formatter` does not return a string.
-
-    See Also
-    --------
-    array_str, array_repr, set_printoptions, get_printoptions
-
-    Notes
-    -----
-    If a formatter is specified for a certain type, the `precision` keyword is
-    ignored for that type.
-
-    Examples
-    --------
-    >>> x = np.array([1e-16,1,2,3])
-    >>> print np.array2string(x, precision=2, separator=',',
-    ...                       suppress_small=True)
-    [ 0., 1., 2., 3.]
-
-    >>> x  = np.arange(3.)
-    >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x})
-    '[0.00 1.00 2.00]'
-
-    >>> x  = np.arange(3)
-    >>> np.array2string(x, formatter={'int':lambda x: hex(x)})
-    '[0x0L 0x1L 0x2L]'
-
-    """
-
-    if a.shape == ():
-        x = a.item()
-        try:
-            lst = a._format(x)
-            msg = "The `_format` attribute is deprecated in Numpy " \
-                  "2.0 and will be removed in 2.1. Use the " \
-                  "`formatter` kw instead."
-            import warnings
-            warnings.warn(msg, DeprecationWarning)
-        except AttributeError:
-            if isinstance(x, tuple):
-                x = _convert_arrays(x)
-            lst = style(x)
-    elif reduce(product, a.shape) == 0:
-        # treat as a null array if any of shape elements == 0
-        lst = "[]"
-    else:
-        lst = _array2string(a, max_line_width, precision, suppress_small,
-                            separator, prefix, formatter=formatter)
-    return lst
-
-def _extendLine(s, line, word, max_line_len, next_line_prefix):
-    if len(line.rstrip()) + len(word.rstrip()) >= max_line_len:
-        s += line.rstrip() + "\n"
-        line = next_line_prefix
-    line += word
-    return s, line
-
-
-def _formatArray(a, format_function, rank, max_line_len,
-                 next_line_prefix, separator, edge_items, summary_insert):
-    """formatArray is designed for two modes of operation:
-
-    1. Full output
-
-    2. Summarized output
-
-    """
-    if rank == 0:
-        obj = a.item()
-        if isinstance(obj, tuple):
-            obj = _convert_arrays(obj)
-        return str(obj)
-
-    if summary_insert and 2*edge_items < len(a):
-        leading_items, trailing_items, summary_insert1 = \
-                       edge_items, edge_items, summary_insert
-    else:
-        leading_items, trailing_items, summary_insert1 = 0, len(a), ""
-
-    if rank == 1:
-        s = ""
-        line = next_line_prefix
-        for i in xrange(leading_items):
-            word = format_function(a[i]) + separator
-            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-
-        if summary_insert1:
-            s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix)
-
-        for i in xrange(trailing_items, 1, -1):
-            word = format_function(a[-i]) + separator
-            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-
-        word = format_function(a[-1])
-        s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
-        s += line + "]\n"
-        s = '[' + s[len(next_line_prefix):]
-    else:
-        s = '['
-        sep = separator.rstrip()
-        for i in xrange(leading_items):
-            if i > 0:
-                s += next_line_prefix
-            s += _formatArray(a[i], format_function, rank-1, max_line_len,
-                              " " + next_line_prefix, separator, edge_items,
-                              summary_insert)
-            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
-
-        if summary_insert1:
-            s += next_line_prefix + summary_insert1 + "\n"
-
-        for i in xrange(trailing_items, 1, -1):
-            if leading_items or i != trailing_items:
-                s += next_line_prefix
-            s += _formatArray(a[-i], format_function, rank-1, max_line_len,
-                              " " + next_line_prefix, separator, edge_items,
-                              summary_insert)
-            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
-        if leading_items or trailing_items > 1:
-            s += next_line_prefix
-        s += _formatArray(a[-1], format_function, rank-1, max_line_len,
-                          " " + next_line_prefix, separator, edge_items,
-                          summary_insert).rstrip()+']\n'
-    return s
-
-class FloatFormat(object):
-    def __init__(self, data, precision, suppress_small, sign=False):
-        self.precision = precision
-        self.suppress_small = suppress_small
-        self.sign = sign
-        self.exp_format = False
-        self.large_exponent = False
-        self.max_str_len = 0
-        try:
-            self.fillFormat(data)
-        except (TypeError, NotImplementedError):
-            # if reduce(data) fails, this instance will not be called, just
-            # instantiated in formatdict.
-            pass
-
-    def fillFormat(self, data):
-        import numeric as _nc
-        errstate = _nc.seterr(all='ignore')
-        try:
-            special = isnan(data) | isinf(data)
-            valid = not_equal(data, 0) & ~special
-            non_zero = absolute(data.compress(valid))
-            if len(non_zero) == 0:
-                max_val = 0.
-                min_val = 0.
-            else:
-                max_val = maximum.reduce(non_zero)
-                min_val = minimum.reduce(non_zero)
-                if max_val >= 1.e8:
-                    self.exp_format = True
-                if not self.suppress_small and (min_val < 0.0001
-                                           or max_val/min_val > 1000.):
-                    self.exp_format = True
-        finally:
-            _nc.seterr(**errstate)
-
-        if self.exp_format:
-            self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100
-            self.max_str_len = 8 + self.precision
-            if self.large_exponent:
-                self.max_str_len += 1
-            if self.sign:
-                format = '%+'
-            else:
-                format = '%'
-            format = format + '%d.%de' % (self.max_str_len, self.precision)
-        else:
-            format = '%%.%df' % (self.precision,)
-            if len(non_zero):
-                precision = max([_digits(x, self.precision, format)
-                                 for x in non_zero])
-            else:
-                precision = 0
-            precision = min(self.precision, precision)
-            self.max_str_len = len(str(int(max_val))) + precision + 2
-            if _nc.any(special):
-                self.max_str_len = max(self.max_str_len,
-                                       len(_nan_str),
-                                       len(_inf_str)+1)
-            if self.sign:
-                format = '%#+'
-            else:
-                format = '%#'
-            format = format + '%d.%df' % (self.max_str_len, precision)
-
-        self.special_fmt = '%%%ds' % (self.max_str_len,)
-        self.format = format
-
-    def __call__(self, x, strip_zeros=True):
-        import numeric as _nc
-        err = _nc.seterr(invalid='ignore')
-        try:
-            if isnan(x):
-                if self.sign:
-                    return self.special_fmt % ('+' + _nan_str,)
-                else:
-                    return self.special_fmt % (_nan_str,)
-            elif isinf(x):
-                if x > 0:
-                    if self.sign:
-                        return self.special_fmt % ('+' + _inf_str,)
-                    else:
-                        return self.special_fmt % (_inf_str,)
-                else:
-                    return self.special_fmt % ('-' + _inf_str,)
-        finally:
-            _nc.seterr(**err)
-
-        s = self.format % x
-        if self.large_exponent:
-            # 3-digit exponent
-            expsign = s[-3]
-            if expsign == '+' or expsign == '-':
-                s = s[1:-2] + '0' + s[-2:]
-        elif self.exp_format:
-            # 2-digit exponent
-            if s[-3] == '0':
-                s = ' ' + s[:-3] + s[-2:]
-        elif strip_zeros:
-            z = s.rstrip('0')
-            s = z + ' '*(len(s)-len(z))
-        return s
-
-
-def _digits(x, precision, format):
-    s = format % x
-    z = s.rstrip('0')
-    return precision - len(s) + len(z)
-
-
-_MAXINT = sys.maxint
-_MININT = -sys.maxint-1
-class IntegerFormat(object):
-    def __init__(self, data):
-        try:
-            max_str_len = max(len(str(maximum.reduce(data))),
-                              len(str(minimum.reduce(data))))
-            self.format = '%' + str(max_str_len) + 'd'
-        except (TypeError, NotImplementedError):
-            # if reduce(data) fails, this instance will not be called, just
-            # instantiated in formatdict.
-            pass
-        except ValueError:
-            # this occurs when everything is NA
-            pass
-
-    def __call__(self, x):
-        if _MININT < x < _MAXINT:
-            return self.format % x
-        else:
-            return "%s" % x
-
-class LongFloatFormat(object):
-    # XXX Have to add something to determine the width to use a la FloatFormat
-    # Right now, things won't line up properly
-    def __init__(self, precision, sign=False):
-        self.precision = precision
-        self.sign = sign
-
-    def __call__(self, x):
-        if isnan(x):
-            if self.sign:
-                return '+' + _nan_str
-            else:
-                return ' ' + _nan_str
-        elif isinf(x):
-            if x > 0:
-                if self.sign:
-                    return '+' + _inf_str
-                else:
-                    return ' ' + _inf_str
-            else:
-                return '-' + _inf_str
-        elif x >= 0:
-            if self.sign:
-                return '+' + format_longfloat(x, self.precision)
-            else:
-                return ' ' + format_longfloat(x, self.precision)
-        else:
-            return format_longfloat(x, self.precision)
-
-
-class LongComplexFormat(object):
-    def __init__(self, precision):
-        self.real_format = LongFloatFormat(precision)
-        self.imag_format = LongFloatFormat(precision, sign=True)
-
-    def __call__(self, x):
-        r = self.real_format(x.real)
-        i = self.imag_format(x.imag)
-        return r + i + 'j'
-
-
-class ComplexFormat(object):
-    def __init__(self, x, precision, suppress_small):
-        self.real_format = FloatFormat(x.real, precision, suppress_small)
-        self.imag_format = FloatFormat(x.imag, precision, suppress_small,
-                                       sign=True)
-
-    def __call__(self, x):
-        r = self.real_format(x.real, strip_zeros=False)
-        i = self.imag_format(x.imag, strip_zeros=False)
-        if not self.imag_format.exp_format:
-            z = i.rstrip('0')
-            i = z + 'j' + ' '*(len(i)-len(z))
-        else:
-            i = i + 'j'
-        return r + i
-
-class DatetimeFormat(object):
-    def __init__(self, x, unit=None,
-                timezone=None, casting='same_kind'):
-        # Get the unit from the dtype
-        if unit is None:
-            if x.dtype.kind == 'M':
-                unit = datetime_data(x.dtype)[0]
-            else:
-                unit = 's'
-
-        # If timezone is default, make it 'local' or 'UTC' based on the unit
-        if timezone is None:
-            # Date units -> UTC, time units -> local
-            if unit in ('Y', 'M', 'W', 'D'):
-                self.timezone = 'UTC'
-            else:
-                self.timezone = 'local'
-        else:
-            self.timezone = timezone
-        self.unit = unit
-        self.casting = casting
-
-    def __call__(self, x):
-        return "'%s'" % datetime_as_string(x,
-                                    unit=self.unit,
-                                    timezone=self.timezone,
-                                    casting=self.casting)
-
-class TimedeltaFormat(object):
-    def __init__(self, data):
-        if data.dtype.kind == 'm':
-            v = data.view('i8')
-            max_str_len = max(len(str(maximum.reduce(v))),
-                              len(str(minimum.reduce(v))))
-            self.format = '%' + str(max_str_len) + 'd'
-
-    def __call__(self, x):
-        return self.format % x.astype('i8')
-
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
deleted file mode 100644
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ /dev/null
@@ -1,2924 +0,0 @@
-######################################################################
-# This is a copy of numpy/core/fromnumeric.py modified for numpypy
-######################################################################
-"""Module containing non-deprecated functions borrowed from Numeric.
-
-"""
-from __future__ import division, absolute_import, print_function
-
-import types
-
-from . import multiarray as mu
-from . import umath as um
-from . import numerictypes as nt
-from .numeric import asarray, array, asanyarray, concatenate
-from . import _methods
-
-
-# functions that are methods
-__all__ = [
-        'alen', 'all', 'alltrue', 'amax', 'amin', 'any', 'argmax',
-        'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip',
-        'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean',
-        'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put',
-        'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_',
-        'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze',
-        'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var',
-        ]
-
-
-try:
-    _gentype = types.GeneratorType
-except AttributeError:
-    _gentype = type(None)
-
-# save away Python sum
-_sum_ = sum
-
-# functions that are now methods
-def _wrapit(obj, method, *args, **kwds):
-    try:
-        wrap = obj.__array_wrap__
-    except AttributeError:
-        wrap = None
-    result = getattr(asarray(obj), method)(*args, **kwds)
-    if wrap:
-        if not isinstance(result, mu.ndarray):
-            result = asarray(result)
-        result = wrap(result)
-    return result
-
-
-def take(a, indices, axis=None, out=None, mode='raise'):
-    """
-    Take elements from an array along an axis.
-
-    This function does the same thing as "fancy" indexing (indexing arrays
-    using arrays); however, it can be easier to use if you need elements
-    along a given axis.
-
-    Parameters
-    ----------
-    a : array_like
-        The source array.
-    indices : array_like
-        The indices of the values to extract.
-
-        .. versionadded:: 1.8.0
-
-        Also allow scalars for indices.
-    axis : int, optional
-        The axis over which to select values. By default, the flattened
-        input array is used.
-    out : ndarray, optional
-        If provided, the result will be placed in this array. It should
-        be of the appropriate shape and dtype.
-    mode : {'raise', 'wrap', 'clip'}, optional
-        Specifies how out-of-bounds indices will behave.
-
-        * 'raise' -- raise an error (default)
-        * 'wrap' -- wrap around
-        * 'clip' -- clip to the range
-
-        'clip' mode means that all indices that are too large are replaced
-        by the index that addresses the last element along that axis. Note
-        that this disables indexing with negative numbers.
-
-    Returns
-    -------
-    subarray : ndarray
-        The returned array has the same type as `a`.
-
-    See Also
-    --------
-    ndarray.take : equivalent method
-
-    Examples
-    --------
-    >>> a = [4, 3, 5, 7, 6, 8]
-    >>> indices = [0, 1, 4]
-    >>> np.take(a, indices)
-    array([4, 3, 6])
-
-    In this example if `a` is an ndarray, "fancy" indexing can be used.
-
-    >>> a = np.array(a)
-    >>> a[indices]
-    array([4, 3, 6])
-
-    If `indices` is not one dimensional, the output also has these dimensions.
-
-    >>> np.take(a, [[0, 1], [2, 3]])
-    array([[4, 3],
-           [5, 7]])
-    """
-    try:
-        take = a.take
-    except AttributeError:
-        return _wrapit(a, 'take', indices, axis, out, mode)
-    return take(indices, axis, out, mode)
-
-
-# not deprecated --- copy if necessary, view otherwise
-def reshape(a, newshape, order='C'):
-    """
-    Gives a new shape to an array without changing its data.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be reshaped.
-    newshape : int or tuple of ints
-        The new shape should be compatible with the original shape. If
-        an integer, then the result will be a 1-D array of that length.
-        One shape dimension can be -1. In this case, the value is inferred
-        from the length of the array and remaining dimensions.
-    order : {'C', 'F', 'A'}, optional
-        Read the elements of `a` using this index order, and place the elements
-        into the reshaped array using this index order.  'C' means to
-        read / write the elements using C-like index order, with the last axis index
-        changing fastest, back to the first axis index changing slowest.  'F'
-        means to read / write the elements using Fortran-like index order, with
-        the first index changing fastest, and the last index changing slowest.
-        Note that the 'C' and 'F' options take no account of the memory layout
-        of the underlying array, and only refer to the order of indexing.  'A'
-        means to read / write the elements in Fortran-like index order if `a` is
-        Fortran *contiguous* in memory, C-like order otherwise.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        This will be a new view object if possible; otherwise, it will
-        be a copy.  Note there is no guarantee of the *memory layout* (C- or
-        Fortran- contiguous) of the returned array.
-
-    See Also
-    --------
-    ndarray.reshape : Equivalent method.
-
-    Notes
-    -----
-    It is not always possible to change the shape of an array without
-    copying the data. If you want an error to be raise if the data is copied,
-    you should assign the new shape to the shape attribute of the array::
-
-     >>> a = np.zeros((10, 2))
-     # A transpose make the array non-contiguous
-     >>> b = a.T
-     # Taking a view makes it possible to modify the shape without modifying the
-     # initial object.
-     >>> c = b.view()
-     >>> c.shape = (20)
-     AttributeError: incompatible shape for a non-contiguous array
-
-    The `order` keyword gives the index ordering both for *fetching* the values
-    from `a`, and then *placing* the values into the output array.  For example,
-    let's say you have an array:
-
-    >>> a = np.arange(6).reshape((3, 2))
-    >>> a
-    array([[0, 1],
-           [2, 3],
-           [4, 5]])
-
-    You can think of reshaping as first raveling the array (using the given
-    index order), then inserting the elements from the raveled array into the
-    new array using the same kind of index ordering as was used for the
-    raveling.
-
-    >>> np.reshape(a, (2, 3)) # C-like index ordering
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.reshape(a, (2, 3), order='F') # Fortran-like index ordering
-    array([[0, 4, 3],
-           [2, 1, 5]])
-    >>> np.reshape(np.ravel(a, order='F'), (2, 3), order='F')
-    array([[0, 4, 3],
-           [2, 1, 5]])
-
-    Examples
-    --------
-    >>> a = np.array([[1,2,3], [4,5,6]])
-    >>> np.reshape(a, 6)
-    array([1, 2, 3, 4, 5, 6])
-    >>> np.reshape(a, 6, order='F')
-    array([1, 4, 2, 5, 3, 6])
-
-    >>> np.reshape(a, (3,-1))       # the unspecified value is inferred to be 2
-    array([[1, 2],
-           [3, 4],
-           [5, 6]])
-    """
-    assert order == 'C'
-    try:
-        reshape = a.reshape
-    except AttributeError:
-        return _wrapit(a, 'reshape', newshape)
-    return reshape(newshape)
-
-
-def choose(a, choices, out=None, mode='raise'):
-    """
-    Construct an array from an index array and a set of arrays to choose from.
-
-    First of all, if confused or uncertain, definitely look at the Examples -
-    in its full generality, this function is less simple than it might
-    seem from the following code description (below ndi =
-    `numpy.lib.index_tricks`):
-
-    ``np.choose(a,c) == np.array([c[a[I]][I] for I in ndi.ndindex(a.shape)])``.
-
-    But this omits some subtleties.  Here is a fully general summary:
-
-    Given an "index" array (`a`) of integers and a sequence of `n` arrays
-    (`choices`), `a` and each choice array are first broadcast, as necessary,
-    to arrays of a common shape; calling these *Ba* and *Bchoices[i], i =
-    0,...,n-1* we have that, necessarily, ``Ba.shape == Bchoices[i].shape``
-    for each `i`.  Then, a new array with shape ``Ba.shape`` is created as
-    follows:
-
-    * if ``mode=raise`` (the default), then, first of all, each element of
-      `a` (and thus `Ba`) must be in the range `[0, n-1]`; now, suppose that
-      `i` (in that range) is the value at the `(j0, j1, ..., jm)` position
-      in `Ba` - then the value at the same position in the new array is the
-      value in `Bchoices[i]` at that same position;
-
-    * if ``mode=wrap``, values in `a` (and thus `Ba`) may be any (signed)
-      integer; modular arithmetic is used to map integers outside the range
-      `[0, n-1]` back into that range; and then the new array is constructed
-      as above;
-
-    * if ``mode=clip``, values in `a` (and thus `Ba`) may be any (signed)
-      integer; negative integers are mapped to 0; values greater than `n-1`
-      are mapped to `n-1`; and then the new array is constructed as above.
-
-    Parameters
-    ----------
-    a : int array
-        This array must contain integers in `[0, n-1]`, where `n` is the number
-        of choices, unless ``mode=wrap`` or ``mode=clip``, in which cases any
-        integers are permissible.
-    choices : sequence of arrays
-        Choice arrays. `a` and all of the choices must be broadcastable to the
-        same shape.  If `choices` is itself an array (not recommended), then
-        its outermost dimension (i.e., the one corresponding to
-        ``choices.shape[0]``) is taken as defining the "sequence".
-    out : array, optional
-        If provided, the result will be inserted into this array. It should
-        be of the appropriate shape and dtype.
-    mode : {'raise' (default), 'wrap', 'clip'}, optional
-        Specifies how indices outside `[0, n-1]` will be treated:
-
-          * 'raise' : an exception is raised
-          * 'wrap' : value becomes value mod `n`
-          * 'clip' : values < 0 are mapped to 0, values > n-1 are mapped to n-1
-
-    Returns
-    -------
-    merged_array : array
-        The merged result.
-
-    Raises
-    ------
-    ValueError: shape mismatch
-        If `a` and each choice array are not all broadcastable to the same
-        shape.
-
-    See Also
-    --------
-    ndarray.choose : equivalent method
-
-    Notes
-    -----
-    To reduce the chance of misinterpretation, even though the following
-    "abuse" is nominally supported, `choices` should neither be, nor be
-    thought of as, a single array, i.e., the outermost sequence-like container
-    should be either a list or a tuple.
-
-    Examples
-    --------
-
-    >>> choices = [[0, 1, 2, 3], [10, 11, 12, 13],
-    ...   [20, 21, 22, 23], [30, 31, 32, 33]]
-    >>> np.choose([2, 3, 1, 0], choices
-    ... # the first element of the result will be the first element of the
-    ... # third (2+1) "array" in choices, namely, 20; the second element
-    ... # will be the second element of the fourth (3+1) choice array, i.e.,
-    ... # 31, etc.
-    ... )
-    array([20, 31, 12,  3])
-    >>> np.choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1)
-    array([20, 31, 12,  3])
-    >>> # because there are 4 choice arrays
-    >>> np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4)
-    array([20,  1, 12,  3])
-    >>> # i.e., 0
-
-    A couple examples illustrating how choose broadcasts:
-
-    >>> a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]]
-    >>> choices = [-10, 10]
-    >>> np.choose(a, choices)
-    array([[ 10, -10,  10],
-           [-10,  10, -10],
-           [ 10, -10,  10]])
-
-    >>> # With thanks to Anne Archibald
-    >>> a = np.array([0, 1]).reshape((2,1,1))
-    >>> c1 = np.array([1, 2, 3]).reshape((1,3,1))
-    >>> c2 = np.array([-1, -2, -3, -4, -5]).reshape((1,1,5))
-    >>> np.choose(a, (c1, c2)) # result is 2x3x5, res[0,:,:]=c1, res[1,:,:]=c2
-    array([[[ 1,  1,  1,  1,  1],
-            [ 2,  2,  2,  2,  2],
-            [ 3,  3,  3,  3,  3]],
-           [[-1, -2, -3, -4, -5],
-            [-1, -2, -3, -4, -5],
-            [-1, -2, -3, -4, -5]]])
-
-    """
-    try:
-        choose = a.choose
-    except AttributeError:
-        return _wrapit(a, 'choose', choices, out=out, mode=mode)
-    return choose(choices, out=out, mode=mode)
-
-
-def repeat(a, repeats, axis=None):
-    """
-    Repeat elements of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    repeats : {int, array of ints}
-        The number of repetitions for each element.  `repeats` is broadcasted
-        to fit the shape of the given axis.
-    axis : int, optional
-        The axis along which to repeat values.  By default, use the
-        flattened input array, and return a flat output array.
-
-    Returns
-    -------
-    repeated_array : ndarray
-        Output array which has the same shape as `a`, except along
-        the given axis.
-
-    See Also
-    --------
-    tile : Tile an array.
-
-    Examples
-    --------
-    >>> x = np.array([[1,2],[3,4]])
-    >>> np.repeat(x, 2)
-    array([1, 1, 2, 2, 3, 3, 4, 4])
-    >>> np.repeat(x, 3, axis=1)
-    array([[1, 1, 1, 2, 2, 2],
-           [3, 3, 3, 4, 4, 4]])
-    >>> np.repeat(x, [1, 2], axis=0)
-    array([[1, 2],
-           [3, 4],
-           [3, 4]])
-
-    """
-    try:
-        repeat = a.repeat
-    except AttributeError:
-        return _wrapit(a, 'repeat', repeats, axis)
-    return repeat(repeats, axis)
-
-
-def put(a, ind, v, mode='raise'):
-    """
-    Replaces specified elements of an array with given values.
-
-    The indexing works on the flattened target array. `put` is roughly
-    equivalent to:
-
-    ::
-
-        a.flat[ind] = v
-
-    Parameters
-    ----------
-    a : ndarray
-        Target array.
-    ind : array_like
-        Target indices, interpreted as integers.
-    v : array_like
-        Values to place in `a` at target indices. If `v` is shorter than
-        `ind` it will be repeated as necessary.
-    mode : {'raise', 'wrap', 'clip'}, optional
-        Specifies how out-of-bounds indices will behave.
-
-        * 'raise' -- raise an error (default)
-        * 'wrap' -- wrap around
-        * 'clip' -- clip to the range
-
-        'clip' mode means that all indices that are too large are replaced
-        by the index that addresses the last element along that axis. Note
-        that this disables indexing with negative numbers.
-
-    See Also
-    --------
-    putmask, place
-
-    Examples
-    --------
-    >>> a = np.arange(5)
-    >>> np.put(a, [0, 2], [-44, -55])
-    >>> a
-    array([-44,   1, -55,   3,   4])
-
-    >>> a = np.arange(5)
-    >>> np.put(a, 22, -5, mode='clip')
-    >>> a
-    array([ 0,  1,  2,  3, -5])
-
-    """
-    return a.put(ind, v, mode)
-
-
-def swapaxes(a, axis1, axis2):
-    """
-    Interchange two axes of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axis1 : int
-        First axis.
-    axis2 : int
-        Second axis.
-
-    Returns
-    -------
-    a_swapped : ndarray
-        If `a` is an ndarray, then a view of `a` is returned; otherwise
-        a new array is created.
-
-    Examples
-    --------
-    >>> x = np.array([[1,2,3]])
-    >>> np.swapaxes(x,0,1)
-    array([[1],
-           [2],
-           [3]])
-
-    >>> x = np.array([[[0,1],[2,3]],[[4,5],[6,7]]])
-    >>> x
-    array([[[0, 1],
-            [2, 3]],
-           [[4, 5],
-            [6, 7]]])
-
-    >>> np.swapaxes(x,0,2)
-    array([[[0, 4],
-            [2, 6]],
-           [[1, 5],
-            [3, 7]]])
-
-    """
-    try:
-        swapaxes = a.swapaxes
-    except AttributeError:
-        return _wrapit(a, 'swapaxes', axis1, axis2)
-    return swapaxes(axis1, axis2)
-
-
-def transpose(a, axes=None):
-    """
-    Permute the dimensions of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axes : list of ints, optional
-        By default, reverse the dimensions, otherwise permute the axes
-        according to the values given.
-
-    Returns
-    -------
-    p : ndarray
-        `a` with its axes permuted.  A view is returned whenever
-        possible.
-
-    See Also
-    --------
-    rollaxis
-
-    Examples
-    --------
-    >>> x = np.arange(4).reshape((2,2))
-    >>> x
-    array([[0, 1],
-           [2, 3]])
-
-    >>> np.transpose(x)
-    array([[0, 2],
-           [1, 3]])
-
-    >>> x = np.ones((1, 2, 3))
-    >>> np.transpose(x, (1, 0, 2)).shape
-    (2, 1, 3)
-
-    """
-    if axes is not None:
-        raise NotImplementedError('No "axes" arg yet.')
-    try:
-        transpose = a.transpose
-    except AttributeError:
-        return _wrapit(a, 'transpose')
-    return transpose()
-
-
-def partition(a, kth, axis=-1, kind='introselect', order=None):
-    """
-    Return a partitioned copy of an array.
-
-    Creates a copy of the array with its elements rearranged in such a way that
-    the value of the element in kth position is in the position it would be in
-    a sorted array. All elements smaller than the kth element are moved before
-    this element and all equal or greater are moved behind it. The ordering of
-    the elements in the two partitions is undefined.
-
-    .. versionadded:: 1.8.0
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be sorted.
-    kth : int or sequence of ints
-        Element index to partition by. The kth value of the element will be in
-        its final sorted position and all smaller elements will be moved before
-        it and all equal or greater elements behind it.
-        The order all elements in the partitions is undefined.
-        If provided with a sequence of kth it will partition all elements
-        indexed by kth  of them into their sorted position at once.
-    axis : int or None, optional
-        Axis along which to sort. If None, the array is flattened before
-        sorting. The default is -1, which sorts along the last axis.
-    kind : {'introselect'}, optional
-        Selection algorithm. Default is 'introselect'.
-    order : list, optional
-        When `a` is a structured array, this argument specifies which fields
-        to compare first, second, and so on.  This list does not need to
-        include all of the fields.
-
-    Returns
-    -------
-    partitioned_array : ndarray
-        Array of the same type and shape as `a`.
-
-    See Also
-    --------
-    ndarray.partition : Method to sort an array in-place.
-    argpartition : Indirect partition.
-    sort : Full sorting
-
-    Notes
-    -----
-    The various selection algorithms are characterized by their average speed,
-    worst case performance, work space size, and whether they are stable. A
-    stable sort keeps items with the same key in the same relative order. The
-    three available algorithms have the following properties:
-
-    ================= ======= ============= ============ =======
-       kind            speed   worst case    work space  stable
-    ================= ======= ============= ============ =======
-    'introselect'        1        O(n)           0         no
-    ================= ======= ============= ============ =======
-
-    All the partition algorithms make temporary copies of the data when
-    partitioning along any but the last axis.  Consequently, partitioning
-    along the last axis is faster and uses less space than partitioning
-    along any other axis.
-
-    The sort order for complex numbers is lexicographic. If both the real
-    and imaginary parts are non-nan then the order is determined by the
-    real parts except when they are equal, in which case the order is
-    determined by the imaginary parts.
-
-    Examples
-    --------
-    >>> a = np.array([3, 4, 2, 1])
-    >>> np.partition(a, 3)
-    array([2, 1, 3, 4])
-
-    >>> np.partition(a, (1, 3))
-    array([1, 2, 3, 4])
-
-    """
-    if axis is None:
-        a = asanyarray(a).flatten()
-        axis = 0
-    else:
-        a = asanyarray(a).copy()
-    a.partition(kth, axis=axis, kind=kind, order=order)
-    return a
-
-
-def argpartition(a, kth, axis=-1, kind='introselect', order=None):
-    """
-    Perform an indirect partition along the given axis using the algorithm
-    specified by the `kind` keyword. It returns an array of indices of the
-    same shape as `a` that index data along the given axis in partitioned
-    order.
-
-    .. versionadded:: 1.8.0
-
-    Parameters
-    ----------
-    a : array_like
-        Array to sort.
-    kth : int or sequence of ints
-        Element index to partition by. The kth element will be in its final
-        sorted position and all smaller elements will be moved before it and
-        all larger elements behind it.
-        The order all elements in the partitions is undefined.
-        If provided with a sequence of kth it will partition all of them into
-        their sorted position at once.
-    axis : int or None, optional
-        Axis along which to sort.  The default is -1 (the last axis). If None,
-        the flattened array is used.
-    kind : {'introselect'}, optional
-        Selection algorithm. Default is 'introselect'
-    order : list, optional
-        When `a` is an array with fields defined, this argument specifies
-        which fields to compare first, second, etc.  Not all fields need be
-        specified.
-
-    Returns
-    -------
-    index_array : ndarray, int
-        Array of indices that partition `a` along the specified axis.
-        In other words, ``a[index_array]`` yields a sorted `a`.
-
-    See Also
-    --------
-    partition : Describes partition algorithms used.
-    ndarray.partition : Inplace partition.
-    argsort : Full indirect sort
-
-    Notes
-    -----
-    See `partition` for notes on the different selection algorithms.
-
-    Examples
-    --------
-    One dimensional array:
-
-    >>> x = np.array([3, 4, 2, 1])
-    >>> x[np.argpartition(x, 3)]
-    array([2, 1, 3, 4])
-    >>> x[np.argpartition(x, (1, 3))]
-    array([1, 2, 3, 4])
-
-    """
-    return a.argpartition(kth, axis, kind=kind, order=order)
-
-
-def sort(a, axis=-1, kind='quicksort', order=None):
-    """
-    Return a sorted copy of an array.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be sorted.
-    axis : int or None, optional
-        Axis along which to sort. If None, the array is flattened before
-        sorting. The default is -1, which sorts along the last axis.
-    kind : {'quicksort', 'mergesort', 'heapsort'}, optional
-        Sorting algorithm. Default is 'quicksort'.
-    order : list, optional
-        When `a` is a structured array, this argument specifies which fields
-        to compare first, second, and so on.  This list does not need to
-        include all of the fields.
-
-    Returns
-    -------
-    sorted_array : ndarray
-        Array of the same type and shape as `a`.
-
-    See Also
-    --------
-    ndarray.sort : Method to sort an array in-place.
-    argsort : Indirect sort.
-    lexsort : Indirect stable sort on multiple keys.
-    searchsorted : Find elements in a sorted array.
-    partition : Partial sort.
-
-    Notes
-    -----
-    The various sorting algorithms are characterized by their average speed,
-    worst case performance, work space size, and whether they are stable. A
-    stable sort keeps items with the same key in the same relative
-    order. The three available algorithms have the following
-    properties:
-
-    =========== ======= ============= ============ =======
-       kind      speed   worst case    work space  stable
-    =========== ======= ============= ============ =======
-    'quicksort'    1     O(n^2)            0          no
-    'mergesort'    2     O(n*log(n))      ~n/2        yes
-    'heapsort'     3     O(n*log(n))       0          no
-    =========== ======= ============= ============ =======
-
-    All the sort algorithms make temporary copies of the data when
-    sorting along any but the last axis.  Consequently, sorting along
-    the last axis is faster and uses less space than sorting along
-    any other axis.
-
-    The sort order for complex numbers is lexicographic. If both the real
-    and imaginary parts are non-nan then the order is determined by the
-    real parts except when they are equal, in which case the order is
-    determined by the imaginary parts.
-
-    Previous to numpy 1.4.0 sorting real and complex arrays containing nan
-    values led to undefined behaviour. In numpy versions >= 1.4.0 nan
-    values are sorted to the end. The extended sort order is:
-
-      * Real: [R, nan]
-      * Complex: [R + Rj, R + nanj, nan + Rj, nan + nanj]
-
-    where R is a non-nan real value. Complex values with the same nan
-    placements are sorted according to the non-nan part if it exists.
-    Non-nan values are sorted as before.
-
-    Examples
-    --------
-    >>> a = np.array([[1,4],[3,1]])
-    >>> np.sort(a)                # sort along the last axis
-    array([[1, 4],
-           [1, 3]])
-    >>> np.sort(a, axis=None)     # sort the flattened array
-    array([1, 1, 3, 4])
-    >>> np.sort(a, axis=0)        # sort along the first axis
-    array([[1, 1],
-           [3, 4]])
-
-    Use the `order` keyword to specify a field to use when sorting a
-    structured array:
-
-    >>> dtype = [('name', 'S10'), ('height', float), ('age', int)]
-    >>> values = [('Arthur', 1.8, 41), ('Lancelot', 1.9, 38),
-    ...           ('Galahad', 1.7, 38)]
-    >>> a = np.array(values, dtype=dtype)       # create a structured array
-    >>> np.sort(a, order='height')                        # doctest: +SKIP
-    array([('Galahad', 1.7, 38), ('Arthur', 1.8, 41),
-           ('Lancelot', 1.8999999999999999, 38)],
-          dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
-
-    Sort by age, then height if ages are equal:
-
-    >>> np.sort(a, order=['age', 'height'])               # doctest: +SKIP
-    array([('Galahad', 1.7, 38), ('Lancelot', 1.8999999999999999, 38),
-           ('Arthur', 1.8, 41)],
-          dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
-
-    """
-    if axis is None:
-        a = asanyarray(a).flatten()
-        axis = 0
-    else:
-        a = asanyarray(a).copy()
-    a.sort(axis, kind, order)
-    return a
-
-
-def argsort(a, axis=-1, kind='quicksort', order=None):
-    """
-    Returns the indices that would sort an array.
-
-    Perform an indirect sort along the given axis using the algorithm specified
-    by the `kind` keyword. It returns an array of indices of the same shape as
-    `a` that index data along the given axis in sorted order.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to sort.
-    axis : int or None, optional
-        Axis along which to sort.  The default is -1 (the last axis). If None,
-        the flattened array is used.
-    kind : {'quicksort', 'mergesort', 'heapsort'}, optional
-        Sorting algorithm.
-    order : list, optional
-        When `a` is an array with fields defined, this argument specifies
-        which fields to compare first, second, etc.  Not all fields need be
-        specified.
-
-    Returns
-    -------
-    index_array : ndarray, int
-        Array of indices that sort `a` along the specified axis.
-        In other words, ``a[index_array]`` yields a sorted `a`.
-
-    See Also
-    --------
-    sort : Describes sorting algorithms used.
-    lexsort : Indirect stable sort with multiple keys.
-    ndarray.sort : Inplace sort.
-    argpartition : Indirect partial sort.
-
-    Notes
-    -----
-    See `sort` for notes on the different sorting algorithms.
-
-    As of NumPy 1.4.0 `argsort` works with real/complex arrays containing
-    nan values. The enhanced sort order is documented in `sort`.
-
-    Examples
-    --------
-    One dimensional array:
-
-    >>> x = np.array([3, 1, 2])
-    >>> np.argsort(x)
-    array([1, 2, 0])
-
-    Two-dimensional array:
-
-    >>> x = np.array([[0, 3], [2, 2]])
-    >>> x
-    array([[0, 3],
-           [2, 2]])
-
-    >>> np.argsort(x, axis=0)
-    array([[0, 1],
-           [1, 0]])
-
-    >>> np.argsort(x, axis=1)
-    array([[0, 1],
-           [0, 1]])
-
-    Sorting with keys:
-
-    >>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '<i4'), ('y', '<i4')])
-    >>> x
-    array([(1, 0), (0, 1)],
-          dtype=[('x', '<i4'), ('y', '<i4')])
-
-    >>> np.argsort(x, order=('x','y'))
-    array([1, 0])
-
-    >>> np.argsort(x, order=('y','x'))
-    array([0, 1])
-
-    """
-    try:
-        argsort = a.argsort
-    except AttributeError:
-        return _wrapit(a, 'argsort', axis, kind, order)
-    return argsort(axis, kind, order)
-
-
-def argmax(a, axis=None):
-    """
-    Indices of the maximum values along an axis.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axis : int, optional
-        By default, the index is into the flattened array, otherwise
-        along the specified axis.
-
-    Returns
-    -------
-    index_array : ndarray of ints
-        Array of indices into the array. It has the same shape as `a.shape`
-        with the dimension along `axis` removed.
-
-    See Also
-    --------
-    ndarray.argmax, argmin
-    amax : The maximum value along a given axis.
-    unravel_index : Convert a flat index into an index tuple.
-
-    Notes
-    -----
-    In case of multiple occurrences of the maximum values, the indices
-    corresponding to the first occurrence are returned.
-
-    Examples
-    --------
-    >>> a = np.arange(6).reshape(2,3)
-    >>> a
-    array([[0, 1, 2],
-           [3, 4, 5]])
-    >>> np.argmax(a)
-    5
-    >>> np.argmax(a, axis=0)
-    array([1, 1, 1])
-    >>> np.argmax(a, axis=1)
-    array([2, 2])
-
-    >>> b = np.arange(6)
-    >>> b[1] = 5
-    >>> b
-    array([0, 5, 2, 3, 4, 5])
-    >>> np.argmax(b) # Only the first occurrence is returned.
-    1
-
-    """
-    assert axis is None
-    try:
-        argmax = a.argmax
-    except AttributeError:
-        return _wrapit(a, 'argmax')
-    return argmax()
-
-
-def argmin(a, axis=None):
-    """
-    Return the indices of the minimum values along an axis.
-
-    See Also
-    --------
-    argmax : Similar function.  Please refer to `numpy.argmax` for detailed
-        documentation.
-
-    """
-    assert axis is None
-    try:
-        argmin = a.argmin
-    except AttributeError:
-        return _wrapit(a, 'argmin')
-    return argmin()
-
-
-def searchsorted(a, v, side='left', sorter=None):
-    """
-    Find indices where elements should be inserted to maintain order.
-
-    Find the indices into a sorted array `a` such that, if the
-    corresponding elements in `v` were inserted before the indices, the
-    order of `a` would be preserved.
-
-    Parameters
-    ----------
-    a : 1-D array_like
-        Input array. If `sorter` is None, then it must be sorted in
-        ascending order, otherwise `sorter` must be an array of indices
-        that sort it.
-    v : array_like
-        Values to insert into `a`.
-    side : {'left', 'right'}, optional
-        If 'left', the index of the first suitable location found is given.
-        If 'right', return the last such index.  If there is no suitable
-        index, return either 0 or N (where N is the length of `a`).
-    sorter : 1-D array_like, optional
-        .. versionadded:: 1.7.0
-        Optional array of integer indices that sort array a into ascending
-        order. They are typically the result of argsort.
-
-    Returns
-    -------
-    indices : array of ints
-        Array of insertion points with the same shape as `v`.
-
-    See Also
-    --------
-    sort : Return a sorted copy of an array.
-    histogram : Produce histogram from 1-D data.
-
-    Notes
-    -----
-    Binary search is used to find the required insertion points.
-
-    As of Numpy 1.4.0 `searchsorted` works with real/complex arrays containing
-    `nan` values. The enhanced sort order is documented in `sort`.
-
-    Examples
-    --------
-    >>> np.searchsorted([1,2,3,4,5], 3)
-    2
-    >>> np.searchsorted([1,2,3,4,5], 3, side='right')
-    3
-    >>> np.searchsorted([1,2,3,4,5], [-10, 10, 2, 3])
-    array([0, 5, 1, 2])
-
-    """
-    try:
-        searchsorted = a.searchsorted
-    except AttributeError:
-        return _wrapit(a, 'searchsorted', v, side, sorter)
-    return searchsorted(v, side, sorter)
-
-
-def resize(a, new_shape):
-    """
-    Return a new array with the specified shape.
-
-    If the new array is larger than the original array, then the new
-    array is filled with repeated copies of `a`.  Note that this behavior
-    is different from a.resize(new_shape) which fills with zeros instead
-    of repeated copies of `a`.
-
-    Parameters
-    ----------
-    a : array_like
-        Array to be resized.
-
-    new_shape : int or tuple of int
-        Shape of resized array.
-
-    Returns
-    -------
-    reshaped_array : ndarray
-        The new array is formed from the data in the old array, repeated
-        if necessary to fill out the required number of elements.  The
-        data are repeated in the order that they are stored in memory.
-

From noreply at buildbot.pypy.org  Thu Oct 31 13:29:36 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 13:29:36 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Call decay_all_counters() in the
 first case promized by its
Message-ID: <20131031122936.806CA1C1309@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67775:fa73334ff040
Date: 2013-10-31 13:05 +0100
http://bitbucket.org/pypy/pypy/changeset/fa73334ff040/

Log:	Call decay_all_counters() in the first case promized by its
	documentation.

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -129,3 +129,7 @@
     def get_index(self, hash):
         "NOT_RPYTHON"
         return hash
+
+    def decay_all_counters(self):
+        "NOT_RPYTHON"
+        pass
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -162,7 +162,6 @@
 
 
 class WarmEnterState(object):
-    THRESHOLD_LIMIT = sys.maxint // 2
 
     def __init__(self, warmrunnerdesc, jitdriver_sd):
         "NOT_RPYTHON"
@@ -316,6 +315,7 @@
         def bound_reached(index, cell, *args):
             if not confirm_enter_jit(*args):
                 return
+            jitcounter.decay_all_counters()
             # start tracing
             from rpython.jit.metainterp.pyjitpl import MetaInterp
             metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -456,7 +456,7 @@
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
               'function_threshold': 1619, # slightly more than one above, also prime
               'trace_eagerness': 200,
-              'decay': 100,
+              'decay': 25,
               'trace_limit': 6000,
               'inlining': 1,
               'loop_longevity': 1000,

From noreply at buildbot.pypy.org  Thu Oct 31 13:29:38 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 13:29:38 +0100 (CET)
Subject: [pypy-commit] pypy default: Baaah. This was testing the base
	MiniMarkGC class, not
Message-ID: <20131031122938.0612C1C1356@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67776:748abf94a70d
Date: 2013-10-31 13:29 +0100
http://bitbucket.org/pypy/pypy/changeset/748abf94a70d/

Log:	Baaah. This was testing the base MiniMarkGC class, not
	IncrementalMiniMarkGC.

diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -1261,6 +1261,20 @@
 class TestIncrementalMiniMarkGC(TestMiniMarkGC):
     gcname = "incminimark"
 
+    class gcpolicy(gc.BasicFrameworkGcPolicy):
+        class transformerclass(shadowstack.ShadowStackFrameworkGCTransformer):
+            from rpython.memory.gc.incminimark import IncrementalMiniMarkGC \
+                                                      as GCClass
+            GC_PARAMS = {'nursery_size': 32*WORD,
+                         'page_size': 16*WORD,
+                         'arena_size': 64*WORD,
+                         'small_request_threshold': 5*WORD,
+                         'large_object': 8*WORD,
+                         'card_page_indices': 4,
+                         'translated_to_c': False,
+                         }
+            root_stack_depth = 200
+
 
 # ________________________________________________________________
 # tagged pointers

From noreply at buildbot.pypy.org  Thu Oct 31 13:43:53 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 13:43:53 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: hg merge default
Message-ID: <20131031124353.6E5F21C0112@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67777:937ac145d398
Date: 2013-10-31 13:42 +0100
http://bitbucket.org/pypy/pypy/changeset/937ac145d398/

Log:	hg merge default

diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -1261,6 +1261,20 @@
 class TestIncrementalMiniMarkGC(TestMiniMarkGC):
     gcname = "incminimark"
 
+    class gcpolicy(gc.BasicFrameworkGcPolicy):
+        class transformerclass(shadowstack.ShadowStackFrameworkGCTransformer):
+            from rpython.memory.gc.incminimark import IncrementalMiniMarkGC \
+                                                      as GCClass
+            GC_PARAMS = {'nursery_size': 32*WORD,
+                         'page_size': 16*WORD,
+                         'arena_size': 64*WORD,
+                         'small_request_threshold': 5*WORD,
+                         'large_object': 8*WORD,
+                         'card_page_indices': 4,
+                         'translated_to_c': False,
+                         }
+            root_stack_depth = 200
+
 
 # ________________________________________________________________
 # tagged pointers

From noreply at buildbot.pypy.org  Thu Oct 31 13:43:54 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 13:43:54 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Add a hook to call the
 decay_all_counters() function after each
Message-ID: <20131031124354.925031C0112@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67778:10ff42dc1b73
Date: 2013-10-31 13:43 +0100
http://bitbucket.org/pypy/pypy/changeset/10ff42dc1b73/

Log:	Add a hook to call the decay_all_counters() function after each
	minor collection. Currently only implemented with incminimark.

diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -369,7 +369,9 @@
         translator = self.translator
         self.layoutbuilder = framework.TransformerLayoutBuilder(translator)
         self.layoutbuilder.delay_encoding()
-        translator._jit2gc = {'layoutbuilder': self.layoutbuilder}
+        if not hasattr(translator, '_jit2gc'):
+            translator._jit2gc = {}
+        translator._jit2gc['layoutbuilder'] = self.layoutbuilder
 
     def _setup_gcclass(self):
         from rpython.memory.gcheader import GCHeaderBuilder
diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -11,7 +11,7 @@
 class JitCounter:
     DEFAULT_SIZE = 4096
 
-    def __init__(self, size=DEFAULT_SIZE):
+    def __init__(self, size=DEFAULT_SIZE, translator=None):
         "NOT_RPYTHON"
         self.size = size
         self.shift = 1
@@ -23,6 +23,14 @@
                                        track_allocation=False)
         self.celltable = [None] * size
         self._nextindex = 0
+        #
+        if translator is not None:
+            def invoke_after_minor_collection():
+                self.decay_all_counters()
+            if not hasattr(translator, '_jit2gc'):
+                translator._jit2gc = {}
+            translator._jit2gc['invoke_after_minor_collection'] = (
+                invoke_after_minor_collection)
 
     def compute_threshold(self, threshold):
         """Return the 'increment' value corresponding to the given number."""
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -207,7 +207,7 @@
         #
         from rpython.jit.metainterp import counter
         if self.cpu.translate_support_code:
-            self.jitcounter = counter.JitCounter()
+            self.jitcounter = counter.JitCounter(translator=translator)
         else:
             self.jitcounter = counter.DeterministicJitCounter()
         #
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1459,6 +1459,9 @@
                     self.get_total_memory_used())
         if self.DEBUG >= 2:
             self.debug_check_consistency()     # expensive!
+        #
+        self.root_walker.finished_minor_collection()
+        #
         debug_stop("gc-minor")
 
 
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -60,6 +60,9 @@
     def _walk_prebuilt_gc(self, callback):
         pass
 
+    def finished_minor_collection(self):
+        pass
+
 
 class BaseDirectGCTest(object):
     GC_PARAMS = {}
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -142,8 +142,11 @@
 
         if hasattr(translator, '_jit2gc'):
             self.layoutbuilder = translator._jit2gc['layoutbuilder']
+            finished_minor_collection = translator._jit2gc.get(
+                'finished_minor_collection', None)
         else:
             self.layoutbuilder = TransformerLayoutBuilder(translator, GCClass)
+            finished_minor_collection = None
         self.layoutbuilder.transformer = self
         self.get_type_id = self.layoutbuilder.get_type_id
 
@@ -167,6 +170,7 @@
 
         gcdata.gc = GCClass(translator.config.translation, **GC_PARAMS)
         root_walker = self.build_root_walker()
+        root_walker.finished_minor_collection_func = finished_minor_collection
         self.root_walker = root_walker
         gcdata.set_query_functions(gcdata.gc)
         gcdata.gc.set_root_walker(root_walker)
@@ -1285,6 +1289,7 @@
 
 class BaseRootWalker(object):
     thread_setup = None
+    finished_minor_collection_func = None
 
     def __init__(self, gctransformer):
         self.gcdata = gctransformer.gcdata
@@ -1322,6 +1327,11 @@
         if collect_stack_root:
             self.walk_stack_roots(collect_stack_root)     # abstract
 
+    def finished_minor_collection(self):
+        func = self.finished_minor_collection_func
+        if func is not None:
+            func()
+
     def need_stacklet_support(self):
         raise Exception("%s does not support stacklets" % (
             self.__class__.__name__,))
diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py
--- a/rpython/memory/gcwrapper.py
+++ b/rpython/memory/gcwrapper.py
@@ -195,6 +195,9 @@
         for obj in self.gcheap._all_prebuilt_gc:
             collect(llmemory.cast_ptr_to_adr(obj._as_ptr()))
 
+    def finished_minor_collection(self):
+        pass
+
 
 class DirectRunLayoutBuilder(gctypelayout.TypeLayoutBuilder):
 
diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -45,6 +45,8 @@
     taggedpointers = False
 
     def setup_class(cls):
+        cls.marker = lltype.malloc(rffi.CArray(lltype.Signed), 1,
+                                   flavor='raw', zero=True)
         funcs0 = []
         funcs2 = []
         cleanups = []
@@ -744,12 +746,18 @@
     def ensure_layoutbuilder(cls, translator):
         jit2gc = getattr(translator, '_jit2gc', None)
         if jit2gc:
+            assert 'finished_minor_collection' in jit2gc
             return jit2gc['layoutbuilder']
+        marker = cls.marker
         GCClass = cls.gcpolicy.transformerclass.GCClass
         layoutbuilder = framework.TransformerLayoutBuilder(translator, GCClass)
         layoutbuilder.delay_encoding()
+
+        def seeme():
+            marker[0] += 1
         translator._jit2gc = {
             'layoutbuilder': layoutbuilder,
+            'finished_minor_collection': seeme,
         }
         return layoutbuilder
 
@@ -768,6 +776,15 @@
                 g()
                 i += 1
             return 0
+
+        if cls.gcname == 'incminimark':
+            marker = cls.marker
+            def cleanup():
+                assert marker[0] > 0
+                marker[0] = 0
+        else:
+            cleanup = None
+
         def fix_graph_of_g(translator):
             from rpython.translator.translator import graphof
             from rpython.flowspace.model import Constant
@@ -788,7 +805,7 @@
                     break
             else:
                 assert 0, "oups, not found"
-        return f, None, fix_graph_of_g
+        return f, cleanup, fix_graph_of_g
 
     def test_do_malloc_operations(self):
         run = self.runner("do_malloc_operations")

From noreply at buildbot.pypy.org  Thu Oct 31 13:56:33 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 13:56:33 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Fix for 64-bit
Message-ID: <20131031125633.365181C0330@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67779:bd9018c25b45
Date: 2013-10-31 13:55 +0100
http://bitbucket.org/pypy/pypy/changeset/bd9018c25b45/

Log:	Fix for 64-bit

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -42,7 +42,7 @@
         """Return the index (< self.size) from a hash value.  This truncates
         the hash to 32 bits, and then keep the *highest* remaining bits.
         Be sure that hash is computed correctly."""
-        return intmask(r_uint32(r_uint(hash) >> self.shift))
+        return intmask(r_uint32(r_uint(r_uint32(hash)) >> self.shift))
     get_index._always_inline_ = True
 
     def fetch_next_index(self):

From noreply at buildbot.pypy.org  Thu Oct 31 14:28:36 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 14:28:36 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Kill dead code now
Message-ID: <20131031132836.6474D1C0F88@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67780:55bf00a5a0a2
Date: 2013-10-31 14:23 +0100
http://bitbucket.org/pypy/pypy/changeset/55bf00a5a0a2/

Log:	Kill dead code now

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -32,22 +32,6 @@
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
 
-def make_greenkey_dict_key(next_instr, is_being_profiled):
-    # use only uints as keys in the jit_cells dict, rather than
-    # a tuple (next_instr, is_being_profiled)
-    return (
-        (next_instr << 1) |
-        r_uint(intmask(is_being_profiled))
-    )
-
-def get_jitcell_at(next_instr, is_being_profiled, bytecode):
-    key = make_greenkey_dict_key(next_instr, is_being_profiled)
-    return bytecode.jit_cells.get(key, None)
-
-def set_jitcell_at(newcell, next_instr, is_being_profiled, bytecode):
-    key = make_greenkey_dict_key(next_instr, is_being_profiled)
-    bytecode.jit_cells[key] = newcell
-
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
     return (bytecode.co_flags & CO_GENERATOR) != 0
@@ -58,8 +42,6 @@
     virtualizables = ['frame']
 
 pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,
-                              get_jitcell_at = get_jitcell_at,
-                              set_jitcell_at = set_jitcell_at,
                               should_unroll_one_iteration =
                               should_unroll_one_iteration,
                               name='pypyjit')
@@ -121,18 +103,6 @@
     return intmask(decr_by)
 
 
-PyCode__initialize = PyCode._initialize
-
-class __extend__(PyCode):
-    __metaclass__ = extendabletype
-
-    def _initialize(self):
-        PyCode__initialize(self)
-        self.jit_cells = {}
-
-    def _cleanup_(self):
-        self.jit_cells = {}
-
 # ____________________________________________________________
 #
 # Public interface

From noreply at buildbot.pypy.org  Thu Oct 31 14:28:37 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 14:28:37 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Yargh
Message-ID: <20131031132837.7B4071C0F88@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67781:4742af107d39
Date: 2013-10-31 14:27 +0100
http://bitbucket.org/pypy/pypy/changeset/4742af107d39/

Log:	Yargh

diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -344,6 +344,7 @@
             while cell is not None:
                 if isinstance(cell, JitCell) and cell.comparekey(*greenargs):
                     break    # found
+                cell = cell.next
             else:
                 # not found. increment the counter
                 if jitcounter.tick(index, increment_threshold):

From noreply at buildbot.pypy.org  Thu Oct 31 14:58:39 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 14:58:39 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: triviality
Message-ID: <20131031135839.41B501C10F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67782:d52de5c7b0f7
Date: 2013-10-31 14:35 +0100
http://bitbucket.org/pypy/pypy/changeset/d52de5c7b0f7/

Log:	triviality

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -75,9 +75,8 @@
         cell = self.celltable[index]
         keep = newcell
         while cell is not None:
-            remove_me = cell.should_remove_jitcell()
             nextcell = cell.next
-            if not remove_me:
+            if not cell.should_remove_jitcell():
                 cell.next = keep
                 keep = cell
             cell = nextcell

From noreply at buildbot.pypy.org  Thu Oct 31 14:58:40 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 14:58:40 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: More of the same
Message-ID: <20131031135840.C0E1E1C10F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67783:17d7a1ae1a69
Date: 2013-10-31 14:52 +0100
http://bitbucket.org/pypy/pypy/changeset/17d7a1ae1a69/

Log:	More of the same

diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -459,6 +459,7 @@
                     if (isinstance(cell, JitCell) and
                             cell.comparekey(*greenargs)):
                         return cell
+                    cell = cell.next
                 return None
 
             @staticmethod
@@ -475,6 +476,7 @@
                     if (isinstance(cell, JitCell) and
                             cell.comparekey(*greenargs)):
                         return cell
+                    cell = cell.next
                 newcell = JitCell(*greenargs)
                 jitcounter.install_new_cell(index, newcell)
                 return newcell

From noreply at buildbot.pypy.org  Thu Oct 31 17:34:50 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 17:34:50 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Turned out to be a bad idea to
 re-enable "don't-inline" functions after
Message-ID: <20131031163450.1EBC71C324E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67784:d1b8053555be
Date: 2013-10-31 17:11 +0100
http://bitbucket.org/pypy/pypy/changeset/d1b8053555be/

Log:	Turned out to be a bad idea to re-enable "don't-inline" functions
	after a short while. Maybe after a long time would make sense, but
	it seems really counter-productive for now. Fixed, and use a single
	word instead of three boolean flags.

diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -124,10 +124,12 @@
         return rffi.cast(lltype.Signed, x)
 
 
+JC_TRACING         = 0x01
+JC_DONT_TRACE_HERE = 0x02
+JC_TEMPORARY       = 0x04
+
 class BaseJitCell(object):
-    tracing = False
-    temporary = False
-    dont_trace_here = chr(0)
+    flags = 0     # JC_xxx flags
     wref_procedure_token = None
     next = None
 
@@ -140,7 +142,10 @@
 
     def set_procedure_token(self, token, tmp=False):
         self.wref_procedure_token = self._makeref(token)
-        self.temporary = tmp
+        if tmp:
+            self.flags |= JC_TEMPORARY
+        else:
+            self.flags &= ~JC_TEMPORARY
 
     def _makeref(self, token):
         assert token is not None
@@ -149,14 +154,9 @@
     def should_remove_jitcell(self):
         if self.get_procedure_token() is not None:
             return False    # don't remove JitCells with a procedure_token
-        if self.tracing:
-            return False    # don't remove JitCells that are being traced
-        if ord(self.dont_trace_here) == 0:
-            return True     # no reason to keep this JitCell
-        else:
-            # decrement dont_trace_here; it will eventually reach zero.
-            self.dont_trace_here = chr(ord(self.dont_trace_here) - 1)
-            return False
+        # don't remove JitCells that are being traced, or JitCells with
+        # the "don't trace here" flag.  Other JitCells can be removed.
+        return (self.flags & (JC_TRACING | JC_DONT_TRACE_HERE)) == 0
 
 # ____________________________________________________________
 
@@ -239,7 +239,7 @@
 
     def disable_noninlinable_function(self, greenkey):
         cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
-        cell.dont_trace_here = chr(20)
+        cell.flags |= JC_DONT_TRACE_HERE
         debug_start("jit-disableinlining")
         loc = self.get_location_str(greenkey)
         debug_print("disabled inlining", loc)
@@ -323,11 +323,11 @@
             if cell is None:
                 cell = JitCell(*greenargs)
                 jitcounter.install_new_cell(index, cell)
-            cell.tracing = True
+            cell.flags |= JC_TRACING
             try:
                 metainterp.compile_and_run_once(jitdriver_sd, *args)
             finally:
-                cell.tracing = False
+                cell.flags &= ~JC_TRACING
 
         def maybe_compile_and_run(increment_threshold, *args):
             """Entry point to the JIT.  Called at the point with the
@@ -353,11 +353,11 @@
 
             # Here, we have found 'cell'.
             #
-            if cell.tracing:
-                # tracing already happening in some outer invocation of
-                # this function. don't trace a second time.
-                return
-            if cell.temporary:
+            if cell.flags & (JC_TRACING | JC_TEMPORARY):
+                if cell.flags & JC_TRACING:
+                    # tracing already happening in some outer invocation of
+                    # this function. don't trace a second time.
+                    return
                 # attached by compile_tmp_callback().  count normally
                 if jitcounter.tick(index, increment_threshold):
                     bound_reached(index, cell, *args)
@@ -500,7 +500,7 @@
             if can_never_inline(*greenargs):
                 return False
             cell = JitCell.get_jitcell(*greenargs)
-            if cell is not None and ord(cell.dont_trace_here) != 0:
+            if cell is not None and (cell.flags & JC_DONT_TRACE_HERE) != 0:
                 return False
             return True
         def can_inline_callable(greenkey):

From noreply at buildbot.pypy.org  Thu Oct 31 17:34:51 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 17:34:51 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Increase the size of the timetable
 cache. It seems to suffer from a bit
Message-ID: <20131031163451.2D49E1C324E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67785:a8ce9a9c7f21
Date: 2013-10-31 17:15 +0100
http://bitbucket.org/pypy/pypy/changeset/a8ce9a9c7f21/

Log:	Increase the size of the timetable cache. It seems to suffer from a
	bit too many collisions when running translate.py (not too
	surprizing but still)

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -9,7 +9,7 @@
 
 
 class JitCounter:
-    DEFAULT_SIZE = 4096
+    DEFAULT_SIZE = 8192
 
     def __init__(self, size=DEFAULT_SIZE, translator=None):
         "NOT_RPYTHON"

From noreply at buildbot.pypy.org  Thu Oct 31 17:34:52 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 17:34:52 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Use systematically unsigned numbers
 for the timetable index.
Message-ID: <20131031163452.534ED1C3258@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67786:78612a2b8a5a
Date: 2013-10-31 17:27 +0100
http://bitbucket.org/pypy/pypy/changeset/78612a2b8a5a/

Log:	Use systematically unsigned numbers for the timetable index.
	Generates a bit simpler code.

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -3,6 +3,7 @@
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
+from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rlib import rstack
 from rpython.rlib.jit import JitDebugInfo, Counters, dont_look_inside
 from rpython.conftest import option
@@ -494,7 +495,7 @@
     rd_virtuals = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
 
-    status = 0
+    status = r_uint(0)
 
     ST_BUSY_FLAG    = 0x01     # if set, busy tracing from the guard
     ST_TYPE_MASK    = 0x06     # mask for the type (TY_xxx)
@@ -531,7 +532,7 @@
                 ty = self.TY_FLOAT
             else:
                 assert 0, box.type
-            self.status = ty | (i << self.ST_SHIFT)
+            self.status = ty | (r_uint(i) << self.ST_SHIFT)
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         if self.must_compile(deadframe, metainterp_sd, jitdriver_sd):
@@ -574,8 +575,8 @@
         else:    # we have a GUARD_VALUE that fails.
             from rpython.rlib.objectmodel import current_object_addr_as_int
 
-            index = self.status >> self.ST_SHIFT
-            typetag = self.status & self.ST_TYPE_MASK
+            index = intmask(self.status >> self.ST_SHIFT)
+            typetag = intmask(self.status & self.ST_TYPE_MASK)
 
             # fetch the actual value of the guard_value, possibly turning
             # it to an integer
diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -1,4 +1,4 @@
-from rpython.rlib.rarithmetic import r_singlefloat, intmask, r_uint
+from rpython.rlib.rarithmetic import r_singlefloat, r_uint
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
@@ -22,7 +22,7 @@
                                        flavor='raw', zero=True,
                                        track_allocation=False)
         self.celltable = [None] * size
-        self._nextindex = 0
+        self._nextindex = r_uint(0)
         #
         if translator is not None:
             def invoke_after_minor_collection():
@@ -42,7 +42,9 @@
         """Return the index (< self.size) from a hash value.  This truncates
         the hash to 32 bits, and then keep the *highest* remaining bits.
         Be sure that hash is computed correctly."""
-        return intmask(r_uint32(r_uint(r_uint32(hash)) >> self.shift))
+        hash32 = r_uint(r_uint32(hash))  # mask off the bits higher than 32
+        index = hash32 >> self.shift     # shift, resulting in a value < size
+        return index                     # return the result as a r_uint
     get_index._always_inline_ = True
 
     def fetch_next_index(self):

From noreply at buildbot.pypy.org  Thu Oct 31 19:06:18 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 19:06:18 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Oups.
Message-ID: <20131031180618.862FF1C0112@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67787:15a55bcf9518
Date: 2013-10-31 19:05 +0100
http://bitbucket.org/pypy/pypy/changeset/15a55bcf9518/

Log:	Oups.

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -143,7 +143,7 @@
         if hasattr(translator, '_jit2gc'):
             self.layoutbuilder = translator._jit2gc['layoutbuilder']
             finished_minor_collection = translator._jit2gc.get(
-                'finished_minor_collection', None)
+                'invoke_after_minor_collection', None)
         else:
             self.layoutbuilder = TransformerLayoutBuilder(translator, GCClass)
             finished_minor_collection = None
diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -746,7 +746,7 @@
     def ensure_layoutbuilder(cls, translator):
         jit2gc = getattr(translator, '_jit2gc', None)
         if jit2gc:
-            assert 'finished_minor_collection' in jit2gc
+            assert 'invoke_after_minor_collection' in jit2gc
             return jit2gc['layoutbuilder']
         marker = cls.marker
         GCClass = cls.gcpolicy.transformerclass.GCClass
@@ -757,7 +757,7 @@
             marker[0] += 1
         translator._jit2gc = {
             'layoutbuilder': layoutbuilder,
-            'finished_minor_collection': seeme,
+            'invoke_after_minor_collection': seeme,
         }
         return layoutbuilder
 

From noreply at buildbot.pypy.org  Thu Oct 31 21:49:09 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 21:49:09 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Adjust "decay" for measured usage
 (translate.py --annotate): massively
Message-ID: <20131031204909.6C7C61C0330@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67788:fd0ea52daeaf
Date: 2013-10-31 21:48 +0100
http://bitbucket.org/pypy/pypy/changeset/fd0ea52daeaf/

Log:	Adjust "decay" for measured usage (translate.py --annotate):
	massively reduce the per-minor-collection count, but not the per-
	new-loop count. Done by only calling decay_all_counters() every 64
	minor collections rather than every time.

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -25,8 +25,15 @@
         self._nextindex = r_uint(0)
         #
         if translator is not None:
+            self._decay_phase = 0
             def invoke_after_minor_collection():
-                self.decay_all_counters()
+                # After 64 minor collections, we call decay_all_counters().
+                # The "--jit decay=N" option measures the amount the
+                # counters are then reduced by.
+                self._decay_phase += 1
+                if self._decay_phase == 64:
+                    self._decay_phase = 0
+                    self.decay_all_counters()
             if not hasattr(translator, '_jit2gc'):
                 translator._jit2gc = {}
             translator._jit2gc['invoke_after_minor_collection'] = (
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -442,7 +442,7 @@
     'threshold': 'number of times a loop has to run for it to become hot',
     'function_threshold': 'number of times a function must run for it to become traced from start',
     'trace_eagerness': 'number of times a guard has to fail before we start compiling a bridge',
-    'decay': 'decay counters at each minor collection (0=none, 1000=max)',
+    'decay': 'amount to regularly decay counters by (0=none, 1000=max)',
     'trace_limit': 'number of recorded operations before we abort tracing with ABORT_TOO_LONG',
     'inlining': 'inline python functions or not (1/0)',
     'loop_longevity': 'a parameter controlling how long loops will be kept before being freed, an estimate',
@@ -456,7 +456,7 @@
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
               'function_threshold': 1619, # slightly more than one above, also prime
               'trace_eagerness': 200,
-              'decay': 25,
+              'decay': 40,
               'trace_limit': 6000,
               'inlining': 1,
               'loop_longevity': 1000,

From noreply at buildbot.pypy.org  Thu Oct 31 22:42:05 2013
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 31 Oct 2013 22:42:05 +0100 (CET)
Subject: [pypy-commit] pypy jit-counter: Translation fix
Message-ID: <20131031214205.E4BB01C0112@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-counter
Changeset: r67789:9fcd5058364e
Date: 2013-10-31 22:41 +0100
http://bitbucket.org/pypy/pypy/changeset/9fcd5058364e/

Log:	Translation fix

diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py
--- a/rpython/jit/metainterp/counter.py
+++ b/rpython/jit/metainterp/counter.py
@@ -25,14 +25,16 @@
         self._nextindex = r_uint(0)
         #
         if translator is not None:
-            self._decay_phase = 0
+            class Glob:
+                step = 0
+            glob = Glob()
             def invoke_after_minor_collection():
                 # After 64 minor collections, we call decay_all_counters().
                 # The "--jit decay=N" option measures the amount the
                 # counters are then reduced by.
-                self._decay_phase += 1
-                if self._decay_phase == 64:
-                    self._decay_phase = 0
+                glob.step += 1
+                if glob.step == 64:
+                    glob.step = 0
                     self.decay_all_counters()
             if not hasattr(translator, '_jit2gc'):
                 translator._jit2gc = {}